Keep follow-ups on the open chat and send prompt history in order (#85)
## Summary - Closes #84. - Same-chat history now defaults on, and existing accounts are switched on. The account toggle stays an opt-out. Follow-ups such as "I don't know, why?" and "put that list in a table" can see the previous turn. - Grounded answers may use the conversation for follow-ups. Citations stay on facts that come from live sources. A pure follow-up skips web search; a follow-up that asks for a price or rating still searches. - Conversation prompts are loaded oldest-first (`created`, then `id`) for the model and for `conversation_details`. The current user turn is dropped from history by matching its text. - Display order of the bubbles is chat_web_app#136. ## Test plan - [ ] `manage.py test chat_backend.tests.test_services_llm chat_backend.tests.test_grounding_search.GroundingPrepassTestCase chat_backend.tests.test_views_conversations chat_backend.tests.test_models.CompanyAndUserTestCase chat_backend.tests.test_migration_graph chat_backend.tests.test_consumers.DatabaseHelperTestCase chat_backend.tests.test_services_rag` - [ ] Apply migration `0036_enable_conversation_context`. - [ ] Ask for a joke, then "I don't know, why?". The reply explains the joke and does not cite an unrelated page. - [ ] Ask for stroller options, then ask for that list in a table with prices and ratings. The table uses the products already named. Missing prices are called out as missing, not the list. - [ ] Turn "Remember this chat" off. The next reply ignores earlier turns. Turn it back on and a follow-up uses them again. - [ ] Reload a multi-turn chat. Bubbles stay oldest at the top. Reviewed-on: #85
This commit was merged in pull request #85.
This commit is contained in:
1 parent
10f02c7cc4
commit
331cff4730
15 files changed
+309
-61
No files matched your search
@@ -143,7 +143,10 @@ def get_messages(conversation_id, prompt, file_string: str = None, file_type: st
|
|||||||
prompt_instance.file_type = file_type
|
prompt_instance.file_type = file_type
|
||||||
prompt_instance.save()
|
prompt_instance.save()
|
||||||
|
|
||||||
for prompt_obj in Prompt.objects.filter(conversation__id=conversation_id):
|
for prompt_obj in (
|
||||||
|
Prompt.objects.filter(conversation__id=conversation_id)
|
||||||
|
.order_by("created", "id")
|
||||||
|
):
|
||||||
messages.append(
|
messages.append(
|
||||||
{
|
{
|
||||||
"content": prompt_obj.message,
|
"content": prompt_obj.message,
|
||||||
|
|||||||
@@ -126,7 +126,10 @@ def get_messages(conversation_id, prompt, file_string: str = None, file_type: st
|
|||||||
prompt_instance.file_type = file_type
|
prompt_instance.file_type = file_type
|
||||||
prompt_instance.save()
|
prompt_instance.save()
|
||||||
|
|
||||||
for prompt_obj in Prompt.objects.filter(conversation__id=conversation_id):
|
for prompt_obj in (
|
||||||
|
Prompt.objects.filter(conversation__id=conversation_id)
|
||||||
|
.order_by("created", "id")
|
||||||
|
):
|
||||||
messages.append(
|
messages.append(
|
||||||
{
|
{
|
||||||
"content": prompt_obj.message,
|
"content": prompt_obj.message,
|
||||||
|
|||||||
@@ -0,0 +1,36 @@
|
|||||||
|
from django.db import migrations, models
|
||||||
|
|
||||||
|
|
||||||
|
def enable_same_chat_history(apps, schema_editor):
|
||||||
|
"""Turn on thread history for accounts still on the old default (#84).
|
||||||
|
|
||||||
|
The column defaulted to false, so an explicit opt-out cannot be told apart
|
||||||
|
from "never touched". Follow-ups were ignoring the open chat. Accounts that
|
||||||
|
still want one-shot answers can turn the setting off again.
|
||||||
|
"""
|
||||||
|
CustomUser = apps.get_model("chat_backend", "CustomUser")
|
||||||
|
CustomUser.objects.filter(use_conversation_context=False).update(
|
||||||
|
use_conversation_context=True
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class Migration(migrations.Migration):
|
||||||
|
|
||||||
|
dependencies = [
|
||||||
|
("chat_backend", "0035_waitlistentry"),
|
||||||
|
]
|
||||||
|
|
||||||
|
operations = [
|
||||||
|
migrations.AlterField(
|
||||||
|
model_name="customuser",
|
||||||
|
name="use_conversation_context",
|
||||||
|
field=models.BooleanField(
|
||||||
|
default=True,
|
||||||
|
help_text=(
|
||||||
|
"When enabled, earlier turns in the open chat are included "
|
||||||
|
"when answering a follow-up. Turn off to answer each message alone."
|
||||||
|
),
|
||||||
|
),
|
||||||
|
),
|
||||||
|
migrations.RunPython(enable_same_chat_history, migrations.RunPython.noop),
|
||||||
|
]
|
||||||
@@ -74,10 +74,10 @@ class CustomUser(AbstractUser):
|
|||||||
default=True, help_text="How the conversations should display"
|
default=True, help_text="How the conversations should display"
|
||||||
)
|
)
|
||||||
use_conversation_context = models.BooleanField(
|
use_conversation_context = models.BooleanField(
|
||||||
default=False,
|
default=True,
|
||||||
help_text=(
|
help_text=(
|
||||||
"When enabled, prior turns in the conversation are used as "
|
"When enabled, earlier turns in the open chat are included "
|
||||||
"LLM/RAG context for a more tailored experience"
|
"when answering a follow-up. Turn off to answer each message alone."
|
||||||
),
|
),
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -525,9 +525,7 @@ class DriveConnection(TimeInfoBase):
|
|||||||
help_text="Required for company connections; optional for personal (#55).",
|
help_text="Required for company connections; optional for personal (#55).",
|
||||||
)
|
)
|
||||||
provider = models.CharField(max_length=32, choices=Provider.choices)
|
provider = models.CharField(max_length=32, choices=Provider.choices)
|
||||||
kind = models.CharField(
|
kind = models.CharField(max_length=16, choices=Kind.choices, default=Kind.PERSONAL)
|
||||||
max_length=16, choices=Kind.choices, default=Kind.PERSONAL
|
|
||||||
)
|
|
||||||
access_token = models.TextField(blank=True, default="")
|
access_token = models.TextField(blank=True, default="")
|
||||||
refresh_token = models.TextField(blank=True, default="")
|
refresh_token = models.TextField(blank=True, default="")
|
||||||
token_expires_at = models.DateTimeField(null=True, blank=True)
|
token_expires_at = models.DateTimeField(null=True, blank=True)
|
||||||
|
|||||||
@@ -13,12 +13,18 @@ ASSISTANT_SYSTEM_PROMPT = (
|
|||||||
)
|
)
|
||||||
|
|
||||||
GROUNDED_ANSWER_INSTRUCTIONS = (
|
GROUNDED_ANSWER_INSTRUCTIONS = (
|
||||||
"You have been given numbered live sources. Answer ONLY from those sources. "
|
"You may have numbered live sources and the conversation so far. "
|
||||||
"Cite source indexes inline like [1] or [2]. "
|
"Use the conversation for follow-ups: explain a previous answer, reformat it "
|
||||||
"If the sources do not settle the question, say so explicitly — do not fill "
|
"(including a table of items already named), and reply to lines such as "
|
||||||
"gaps from memory or training data. "
|
'"I don\'t know, why?". Do not say that list or answer is missing when it '
|
||||||
|
"is already in the conversation. "
|
||||||
|
"Cite source indexes inline like [1] or [2] only for facts taken from live sources. "
|
||||||
|
"If a new external fact (price, rating, date, news) is in neither the conversation "
|
||||||
|
"nor the sources, say that specific fact is unavailable, and still complete the "
|
||||||
|
"part the conversation already answers. "
|
||||||
"When sources conflict, prefer the most recent dated source. "
|
"When sources conflict, prefer the most recent dated source. "
|
||||||
"Never state a date, number, name, or event that does not appear in the sources."
|
"Never invent a date, number, name, or event that is in neither the conversation "
|
||||||
|
"nor the sources."
|
||||||
)
|
)
|
||||||
|
|
||||||
RETRIEVAL_FAILED_MESSAGE = (
|
RETRIEVAL_FAILED_MESSAGE = (
|
||||||
|
|||||||
@@ -88,8 +88,8 @@ async def prepare_grounded_chat(
|
|||||||
``on_status(stage, detail)`` emits live activity frames (#96) when provided;
|
``on_status(stage, detail)`` emits live activity frames (#96) when provided;
|
||||||
otherwise uses the context-bound emitter from :mod:`status_context`.
|
otherwise uses the context-bound emitter from :mod:`status_context`.
|
||||||
|
|
||||||
``use_conversation_context`` gates prior-turn history in the LLM prompt
|
``use_conversation_context`` gates prior-turn history in the LLM prompt.
|
||||||
(#33); default off (opt-in).
|
The account setting defaults on so follow-ups see this chat (#84).
|
||||||
"""
|
"""
|
||||||
gen_kwargs = {"use_conversation_context": use_conversation_context}
|
gen_kwargs = {"use_conversation_context": use_conversation_context}
|
||||||
internet = getattr(settings, "ALLOW_INTERNET_ACCESS", False)
|
internet = getattr(settings, "ALLOW_INTERNET_ACCESS", False)
|
||||||
@@ -104,7 +104,9 @@ async def prepare_grounded_chat(
|
|||||||
)
|
)
|
||||||
|
|
||||||
await _emit(on_status, "evaluating")
|
await _emit(on_status, "evaluating")
|
||||||
decision = await grounding_decider.decide_async(message)
|
# Prior turns exist once the current user message is not the only row.
|
||||||
|
has_history = bool(use_conversation_context) and len(messages) > 1
|
||||||
|
decision = await grounding_decider.decide_async(message, has_history=has_history)
|
||||||
|
|
||||||
if not decision.needs_retrieval:
|
if not decision.needs_retrieval:
|
||||||
await _emit(on_status, "refining")
|
await _emit(on_status, "refining")
|
||||||
|
|||||||
@@ -43,6 +43,28 @@ _TEMPORAL_PATTERNS = (
|
|||||||
_TEMPORAL_RE = re.compile("|".join(_TEMPORAL_PATTERNS), re.IGNORECASE)
|
_TEMPORAL_RE = re.compile("|".join(_TEMPORAL_PATTERNS), re.IGNORECASE)
|
||||||
_YEAR_RE = re.compile(r"\b((?:19|20)\d{2})\b")
|
_YEAR_RE = re.compile(r"\b((?:19|20)\d{2})\b")
|
||||||
|
|
||||||
|
# Pure follow-ups should reuse the open chat, not a fresh web search (#84).
|
||||||
|
_PURE_FOLLOWUP_RE = re.compile(
|
||||||
|
r"("
|
||||||
|
r"\bi don'?t know\b"
|
||||||
|
r"|^\s*why\b"
|
||||||
|
r"|\bwhy\s*\??\s*$"
|
||||||
|
r"|\bwhat do you mean\b"
|
||||||
|
r"|\bexplain\b"
|
||||||
|
r"|\b(that|this|the|your) (list|joke|answer|setup|punchline)\b"
|
||||||
|
r"|\bmake (that|this|it|them)\b"
|
||||||
|
r"|\bput (that|this|it|them)\b"
|
||||||
|
r"|\bin a table\b"
|
||||||
|
r"|\binto a table\b"
|
||||||
|
r")",
|
||||||
|
re.IGNORECASE,
|
||||||
|
)
|
||||||
|
# These still need live sources even when the sentence points at earlier turns.
|
||||||
|
_NEW_EXTERNAL_FACT_RE = re.compile(
|
||||||
|
r"\b(price|prices|rating|ratings|cost|costs|review|reviews|cheapest)\b",
|
||||||
|
re.IGNORECASE,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
@dataclass
|
@dataclass
|
||||||
class GroundingDecision:
|
class GroundingDecision:
|
||||||
@@ -107,6 +129,28 @@ message). Example: "did Taylor Swift get married" →
|
|||||||
source="prepass",
|
source="prepass",
|
||||||
)
|
)
|
||||||
|
|
||||||
|
def followup_prepass(
|
||||||
|
self, prompt: str, *, has_history: bool
|
||||||
|
) -> GroundingDecision | None:
|
||||||
|
"""Skip web search when the line only continues the open chat (#84)."""
|
||||||
|
if not has_history:
|
||||||
|
return None
|
||||||
|
text = (prompt or "").strip()
|
||||||
|
if not text or not _PURE_FOLLOWUP_RE.search(text):
|
||||||
|
return None
|
||||||
|
if _NEW_EXTERNAL_FACT_RE.search(text):
|
||||||
|
return None
|
||||||
|
temporal = self.temporal_prepass(text)
|
||||||
|
if temporal is not None and temporal.needs_retrieval:
|
||||||
|
return None
|
||||||
|
return GroundingDecision(
|
||||||
|
needs_retrieval=False,
|
||||||
|
reason="follow-up on the current conversation",
|
||||||
|
queries=[],
|
||||||
|
temporal=False,
|
||||||
|
source="prepass",
|
||||||
|
)
|
||||||
|
|
||||||
def _parse(self, raw: str, fallback_query: str) -> GroundingDecision:
|
def _parse(self, raw: str, fallback_query: str) -> GroundingDecision:
|
||||||
text = (raw or "").strip()
|
text = (raw or "").strip()
|
||||||
# Strip markdown fences if the small model wraps JSON.
|
# Strip markdown fences if the small model wraps JSON.
|
||||||
@@ -144,7 +188,12 @@ message). Example: "did Taylor Swift get married" →
|
|||||||
source="model",
|
source="model",
|
||||||
)
|
)
|
||||||
|
|
||||||
async def decide_async(self, prompt: str) -> GroundingDecision:
|
async def decide_async(
|
||||||
|
self, prompt: str, *, has_history: bool = False
|
||||||
|
) -> GroundingDecision:
|
||||||
|
follow = self.followup_prepass(prompt, has_history=has_history)
|
||||||
|
if follow is not None:
|
||||||
|
return follow
|
||||||
pre = self.temporal_prepass(prompt)
|
pre = self.temporal_prepass(prompt)
|
||||||
# Even on temporal prepass, ask the model for better queries when possible.
|
# Even on temporal prepass, ask the model for better queries when possible.
|
||||||
try:
|
try:
|
||||||
@@ -173,7 +222,10 @@ message). Example: "did Taylor Swift get married" →
|
|||||||
)
|
)
|
||||||
return decision
|
return decision
|
||||||
|
|
||||||
def decide(self, prompt: str) -> GroundingDecision:
|
def decide(self, prompt: str, *, has_history: bool = False) -> GroundingDecision:
|
||||||
|
follow = self.followup_prepass(prompt, has_history=has_history)
|
||||||
|
if follow is not None:
|
||||||
|
return follow
|
||||||
pre = self.temporal_prepass(prompt)
|
pre = self.temporal_prepass(prompt)
|
||||||
try:
|
try:
|
||||||
raw = self.chain.invoke({"prompt": prompt})
|
raw = self.chain.invoke({"prompt": prompt})
|
||||||
|
|||||||
@@ -173,16 +173,12 @@ Response:"""
|
|||||||
block (when grounded) is never trimmed.
|
block (when grounded) is never trimmed.
|
||||||
"""
|
"""
|
||||||
sources = self.sources_block or kwargs.get("sources_block", "") or ""
|
sources = self.sources_block or kwargs.get("sources_block", "") or ""
|
||||||
use_conversation_context = bool(
|
use_conversation_context = bool(kwargs.get("use_conversation_context", False))
|
||||||
kwargs.get("use_conversation_context", False)
|
|
||||||
)
|
|
||||||
history_text = ""
|
history_text = ""
|
||||||
if use_conversation_context:
|
if use_conversation_context:
|
||||||
reserved = (
|
reserved = (
|
||||||
estimate_tokens(ASSISTANT_SYSTEM_PROMPT)
|
estimate_tokens(ASSISTANT_SYSTEM_PROMPT)
|
||||||
+ estimate_tokens(
|
+ estimate_tokens(GROUNDED_ANSWER_INSTRUCTIONS if self.grounded else "")
|
||||||
GROUNDED_ANSWER_INSTRUCTIONS if self.grounded else ""
|
|
||||||
)
|
|
||||||
+ estimate_tokens(sources)
|
+ estimate_tokens(sources)
|
||||||
+ estimate_tokens(query)
|
+ estimate_tokens(query)
|
||||||
+ 256 # response headroom / instructions
|
+ 256 # response headroom / instructions
|
||||||
@@ -203,8 +199,9 @@ Response:"""
|
|||||||
and str(getattr(m, "content", "")).startswith("Live sources:")
|
and str(getattr(m, "content", "")).startswith("Live sources:")
|
||||||
)
|
)
|
||||||
]
|
]
|
||||||
# Exclude the latest user turn from history (it's in {query}).
|
# Current user turn is passed as {query}. Drop that text, not
|
||||||
prior = clean[:-1] if clean else []
|
# whichever row happens to be last.
|
||||||
|
prior = prior_turns(clean, query)
|
||||||
windowed = window_history(
|
windowed = window_history(
|
||||||
prior, budget_tokens=history_budget, reserved_tokens=0
|
prior, budget_tokens=history_budget, reserved_tokens=0
|
||||||
)
|
)
|
||||||
@@ -220,6 +217,33 @@ Response:"""
|
|||||||
yield chunk
|
yield chunk
|
||||||
|
|
||||||
|
|
||||||
|
def message_text(message) -> str:
|
||||||
|
text = getattr(message, "content", "") or ""
|
||||||
|
if not isinstance(text, str):
|
||||||
|
text = str(text)
|
||||||
|
return text
|
||||||
|
|
||||||
|
|
||||||
|
def prior_turns(messages, query: str) -> list:
|
||||||
|
"""Drop the current user turn from history.
|
||||||
|
|
||||||
|
Match the query text, newest hit first, so an unordered transcript cannot
|
||||||
|
drop a different turn just because it sits at the end of the list (#84).
|
||||||
|
If the query is not in the list, keep every turn.
|
||||||
|
"""
|
||||||
|
if not messages:
|
||||||
|
return []
|
||||||
|
needle = (query or "").strip()
|
||||||
|
for index in range(len(messages) - 1, -1, -1):
|
||||||
|
message = messages[index]
|
||||||
|
if (
|
||||||
|
getattr(message, "type", "") == "human"
|
||||||
|
and message_text(message).strip() == needle
|
||||||
|
):
|
||||||
|
return list(messages[:index]) + list(messages[index + 1 :])
|
||||||
|
return list(messages)
|
||||||
|
|
||||||
|
|
||||||
def build_chat_service(
|
def build_chat_service(
|
||||||
*,
|
*,
|
||||||
model_name: str | None,
|
model_name: str | None,
|
||||||
|
|||||||
@@ -1,9 +1,11 @@
|
|||||||
from unittest import mock
|
|
||||||
import json
|
import json
|
||||||
|
from datetime import timedelta
|
||||||
|
from unittest import mock
|
||||||
|
|
||||||
from asgiref.sync import sync_to_async
|
from asgiref.sync import sync_to_async
|
||||||
from channels.testing import WebsocketCommunicator
|
from channels.testing import WebsocketCommunicator
|
||||||
from django.test import TestCase, TransactionTestCase, override_settings
|
from django.test import TestCase, TransactionTestCase, override_settings
|
||||||
|
from django.utils import timezone
|
||||||
from langchain_core.messages import AIMessage, HumanMessage
|
from langchain_core.messages import AIMessage, HumanMessage
|
||||||
from parameterized import parameterized
|
from parameterized import parameterized
|
||||||
|
|
||||||
@@ -47,9 +49,7 @@ class DatabaseHelperTestCase(TransactionTestCase):
|
|||||||
|
|
||||||
@parameterized.expand([("websocket", consumers), ("langgraph", consumers_graph)])
|
@parameterized.expand([("websocket", consumers), ("langgraph", consumers_graph)])
|
||||||
async def test_get_workspace(self, _name, module):
|
async def test_get_workspace(self, _name, module):
|
||||||
workspace = await module.get_workspace(
|
workspace = await module.get_workspace(self.conversation.id, user=self.user)
|
||||||
self.conversation.id, user=self.user
|
|
||||||
)
|
|
||||||
|
|
||||||
self.assertEqual(workspace.id, self.workspace.id)
|
self.assertEqual(workspace.id, self.workspace.id)
|
||||||
|
|
||||||
@@ -99,6 +99,29 @@ class DatabaseHelperTestCase(TransactionTestCase):
|
|||||||
[type(message) for message in messages], [AIMessage, HumanMessage]
|
[type(message) for message in messages], [AIMessage, HumanMessage]
|
||||||
)
|
)
|
||||||
|
|
||||||
|
@parameterized.expand([("websocket", consumers), ("langgraph", consumers_graph)])
|
||||||
|
async def test_get_messages_orders_by_created_not_insert_order(self, _name, module):
|
||||||
|
now = timezone.now()
|
||||||
|
await sync_to_async(Prompt.objects.create)(
|
||||||
|
conversation=self.conversation,
|
||||||
|
message="newer answer",
|
||||||
|
user_created=False,
|
||||||
|
created=now,
|
||||||
|
)
|
||||||
|
await sync_to_async(Prompt.objects.create)(
|
||||||
|
conversation=self.conversation,
|
||||||
|
message="older question",
|
||||||
|
user_created=True,
|
||||||
|
created=now - timedelta(minutes=5),
|
||||||
|
)
|
||||||
|
|
||||||
|
messages, _prompt = await module.get_messages(self.conversation.id, "follow up")
|
||||||
|
|
||||||
|
self.assertEqual(
|
||||||
|
[message.content for message in messages],
|
||||||
|
["older question", "newer answer", "follow up"],
|
||||||
|
)
|
||||||
|
|
||||||
async def test_get_messages_stores_attachment_and_inlines_csv(self):
|
async def test_get_messages_stores_attachment_and_inlines_csv(self):
|
||||||
messages, prompt_instance = await consumers.get_messages(
|
messages, prompt_instance = await consumers.get_messages(
|
||||||
self.conversation.id, "analyze this", b"name,sales\nalice,10\n", "csv"
|
self.conversation.id, "analyze this", b"name,sales\nalice,10\n", "csv"
|
||||||
@@ -587,13 +610,16 @@ class WebSocketReceiveGuardTestCase(TransactionTestCase):
|
|||||||
connected, _ = await communicator.connect()
|
connected, _ = await communicator.connect()
|
||||||
self.assertTrue(connected)
|
self.assertTrue(connected)
|
||||||
|
|
||||||
with mock.patch(
|
with (
|
||||||
"chat_backend.consumers.title_generator.generate_async",
|
mock.patch(
|
||||||
new_callable=mock.AsyncMock,
|
"chat_backend.consumers.title_generator.generate_async",
|
||||||
) as title_chat, mock.patch(
|
new_callable=mock.AsyncMock,
|
||||||
"chat_backend.consumers_graph.title_generator.generate_async",
|
) as title_chat,
|
||||||
new_callable=mock.AsyncMock,
|
mock.patch(
|
||||||
) as title_graph:
|
"chat_backend.consumers_graph.title_generator.generate_async",
|
||||||
|
new_callable=mock.AsyncMock,
|
||||||
|
) as title_graph,
|
||||||
|
):
|
||||||
await communicator.send_json_to({"type": "ping", "email": "a@b.com"})
|
await communicator.send_json_to({"type": "ping", "email": "a@b.com"})
|
||||||
# No reply expected; give the event loop a tick.
|
# No reply expected; give the event loop a tick.
|
||||||
self.assertTrue(await communicator.receive_nothing(timeout=0.2))
|
self.assertTrue(await communicator.receive_nothing(timeout=0.2))
|
||||||
@@ -612,13 +638,16 @@ class WebSocketReceiveGuardTestCase(TransactionTestCase):
|
|||||||
connected, _ = await communicator.connect()
|
connected, _ = await communicator.connect()
|
||||||
self.assertTrue(connected)
|
self.assertTrue(connected)
|
||||||
|
|
||||||
with mock.patch(
|
with (
|
||||||
"chat_backend.consumers.title_generator.generate_async",
|
mock.patch(
|
||||||
new_callable=mock.AsyncMock,
|
"chat_backend.consumers.title_generator.generate_async",
|
||||||
) as title_chat, mock.patch(
|
new_callable=mock.AsyncMock,
|
||||||
"chat_backend.consumers_graph.title_generator.generate_async",
|
) as title_chat,
|
||||||
new_callable=mock.AsyncMock,
|
mock.patch(
|
||||||
) as title_graph:
|
"chat_backend.consumers_graph.title_generator.generate_async",
|
||||||
|
new_callable=mock.AsyncMock,
|
||||||
|
) as title_graph,
|
||||||
|
):
|
||||||
await communicator.send_json_to(
|
await communicator.send_json_to(
|
||||||
{"message": " ", "email": "a@b.com", "conversation_id": None}
|
{"message": " ", "email": "a@b.com", "conversation_id": None}
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -86,6 +86,25 @@ class GroundingPrepassTestCase(SimpleTestCase):
|
|||||||
self.assertEqual(decision.source, "fail_open")
|
self.assertEqual(decision.source, "fail_open")
|
||||||
self.assertEqual(decision.queries, ["fallback query"])
|
self.assertEqual(decision.queries, ["fallback query"])
|
||||||
|
|
||||||
|
def test_pure_followup_skips_retrieval_without_calling_model(self):
|
||||||
|
self.decider.chain = mock.Mock()
|
||||||
|
decision = self.decider.decide("I don't know. Why?", has_history=True)
|
||||||
|
self.assertFalse(decision.needs_retrieval)
|
||||||
|
self.assertEqual(decision.reason, "follow-up on the current conversation")
|
||||||
|
self.decider.chain.invoke.assert_not_called()
|
||||||
|
|
||||||
|
def test_followup_without_history_does_not_short_circuit(self):
|
||||||
|
self.assertIsNone(
|
||||||
|
self.decider.followup_prepass("I don't know. Why?", has_history=False)
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_price_followup_still_allows_retrieval(self):
|
||||||
|
decision = self.decider.followup_prepass(
|
||||||
|
"Can you make that list for me and add the prices and ratings in a table",
|
||||||
|
has_history=True,
|
||||||
|
)
|
||||||
|
self.assertIsNone(decision)
|
||||||
|
|
||||||
def test_decide_fail_open_on_llm_error(self):
|
def test_decide_fail_open_on_llm_error(self):
|
||||||
self.decider.chain = mock.Mock()
|
self.decider.chain = mock.Mock()
|
||||||
self.decider.chain.invoke.side_effect = RuntimeError("ollama down")
|
self.decider.chain.invoke.side_effect = RuntimeError("ollama down")
|
||||||
|
|||||||
@@ -6,9 +6,11 @@ from django.test import TestCase
|
|||||||
class MigrationGraphTests(TestCase):
|
class MigrationGraphTests(TestCase):
|
||||||
def test_chat_backend_has_a_single_leaf(self):
|
def test_chat_backend_has_a_single_leaf(self):
|
||||||
loader = MigrationLoader(connection, ignore_no_migrations=True)
|
loader = MigrationLoader(connection, ignore_no_migrations=True)
|
||||||
leaves = [name for app, name in loader.graph.leaf_nodes() if app == "chat_backend"]
|
leaves = [
|
||||||
self.assertEqual(leaves, ["0035_waitlistentry"])
|
name for app, name in loader.graph.leaf_nodes() if app == "chat_backend"
|
||||||
|
]
|
||||||
|
self.assertEqual(leaves, ["0036_enable_conversation_context"])
|
||||||
|
|
||||||
node = loader.graph.nodes[("chat_backend", "0035_waitlistentry")]
|
node = loader.graph.nodes[("chat_backend", "0036_enable_conversation_context")]
|
||||||
parents = [name for app, name in node.dependencies if app == "chat_backend"]
|
parents = [name for app, name in node.dependencies if app == "chat_backend"]
|
||||||
self.assertEqual(parents, ["0034_customuser_use_conversation_context"])
|
self.assertEqual(parents, ["0035_waitlistentry"])
|
||||||
@@ -81,7 +81,7 @@ class CompanyAndUserTestCase(TestCase):
|
|||||||
self.assertFalse(user.deleted)
|
self.assertFalse(user.deleted)
|
||||||
self.assertFalse(user.has_signed_tos)
|
self.assertFalse(user.has_signed_tos)
|
||||||
self.assertTrue(user.conversation_order)
|
self.assertTrue(user.conversation_order)
|
||||||
self.assertFalse(user.use_conversation_context)
|
self.assertTrue(user.use_conversation_context)
|
||||||
|
|
||||||
|
|
||||||
class ConversationAndPromptTestCase(TestCase):
|
class ConversationAndPromptTestCase(TestCase):
|
||||||
|
|||||||
@@ -1,7 +1,12 @@
|
|||||||
from django.test import SimpleTestCase
|
from django.test import SimpleTestCase
|
||||||
from langchain_core.messages import AIMessage, HumanMessage
|
from langchain_core.messages import AIMessage, HumanMessage
|
||||||
|
|
||||||
from chat_backend.services.llm_service import AsyncLLMService, SyncLLMService
|
from chat_backend.services.assistant_identity import GROUNDED_ANSWER_INSTRUCTIONS
|
||||||
|
from chat_backend.services.llm_service import (
|
||||||
|
AsyncLLMService,
|
||||||
|
SyncLLMService,
|
||||||
|
prior_turns,
|
||||||
|
)
|
||||||
from chat_backend.services.prompt_budget import format_history, window_history
|
from chat_backend.services.prompt_budget import format_history, window_history
|
||||||
|
|
||||||
from .fakes import FakeChain
|
from .fakes import FakeChain
|
||||||
@@ -36,7 +41,7 @@ class AsyncLLMServiceTestCase(SimpleTestCase):
|
|||||||
|
|
||||||
async def test_generate_response_sends_single_history_window(self):
|
async def test_generate_response_sends_single_history_window(self):
|
||||||
self.service.conversation_chain = FakeChain(chunks=["ok"])
|
self.service.conversation_chain = FakeChain(chunks=["ok"])
|
||||||
messages = conversation(4) # 8 messages
|
messages = conversation(4) + [HumanMessage(content="latest")]
|
||||||
|
|
||||||
async for _ in self.service.generate_response(
|
async for _ in self.service.generate_response(
|
||||||
messages, "latest", 1, use_conversation_context=True
|
messages, "latest", 1, use_conversation_context=True
|
||||||
@@ -47,10 +52,33 @@ class AsyncLLMServiceTestCase(SimpleTestCase):
|
|||||||
self.assertEqual(payload["query"], "latest")
|
self.assertEqual(payload["query"], "latest")
|
||||||
# Latest user turn is in {query}; history is prior turns only, once.
|
# Latest user turn is in {query}; history is prior turns only, once.
|
||||||
self.assertIn("history", payload)
|
self.assertIn("history", payload)
|
||||||
|
self.assertNotIn("User: latest", payload["history"])
|
||||||
|
self.assertIn("AI: answer 3", payload["history"])
|
||||||
self.assertNotIn("recent_conversation", payload)
|
self.assertNotIn("recent_conversation", payload)
|
||||||
self.assertNotIn("conversation", payload)
|
self.assertNotIn("conversation", payload)
|
||||||
# 8 messages → drop last (query) → 7 prior lines max in window.
|
self.assertLessEqual(len(payload["history"].splitlines()), 8)
|
||||||
self.assertLessEqual(len(payload["history"].splitlines()), 7)
|
|
||||||
|
async def test_history_drops_matching_turn_even_if_it_is_not_last(self):
|
||||||
|
self.service.conversation_chain = FakeChain(chunks=["ok"])
|
||||||
|
# Unordered transcript: current question is first, older turns follow.
|
||||||
|
messages = [
|
||||||
|
HumanMessage(content="I don't know. Why?"),
|
||||||
|
AIMessage(content="Why did the meditation instructor bring a ladder?"),
|
||||||
|
HumanMessage(content="Tell me a joke"),
|
||||||
|
]
|
||||||
|
|
||||||
|
async for _ in self.service.generate_response(
|
||||||
|
messages,
|
||||||
|
"I don't know. Why?",
|
||||||
|
1,
|
||||||
|
use_conversation_context=True,
|
||||||
|
):
|
||||||
|
pass
|
||||||
|
|
||||||
|
history = self.service.conversation_chain.calls[0]["history"]
|
||||||
|
self.assertNotIn("I don't know", history)
|
||||||
|
self.assertIn("Tell me a joke", history)
|
||||||
|
self.assertIn("meditation instructor", history)
|
||||||
|
|
||||||
async def test_generate_response_skips_history_when_context_disabled(self):
|
async def test_generate_response_skips_history_when_context_disabled(self):
|
||||||
self.service.conversation_chain = FakeChain(chunks=["ok"])
|
self.service.conversation_chain = FakeChain(chunks=["ok"])
|
||||||
@@ -73,7 +101,9 @@ class AsyncLLMServiceTestCase(SimpleTestCase):
|
|||||||
self.assertEqual(self.service.conversation_chain.calls[0]["history"], "")
|
self.assertEqual(self.service.conversation_chain.calls[0]["history"], "")
|
||||||
|
|
||||||
async def test_grounded_service_includes_sources(self):
|
async def test_grounded_service_includes_sources(self):
|
||||||
service = AsyncLLMService(grounded=True, sources_block='[1] "T" — x.com — undated')
|
service = AsyncLLMService(
|
||||||
|
grounded=True, sources_block='[1] "T" — x.com — undated'
|
||||||
|
)
|
||||||
service.conversation_chain = FakeChain(chunks=["ok"])
|
service.conversation_chain = FakeChain(chunks=["ok"])
|
||||||
|
|
||||||
async for _ in service.generate_response(
|
async for _ in service.generate_response(
|
||||||
@@ -90,6 +120,24 @@ class SyncLLMServiceTestCase(SimpleTestCase):
|
|||||||
self.assertIsNotNone(SyncLLMService())
|
self.assertIsNotNone(SyncLLMService())
|
||||||
|
|
||||||
|
|
||||||
|
class PriorTurnsTestCase(SimpleTestCase):
|
||||||
|
def test_grounded_instructions_keep_the_conversation(self):
|
||||||
|
self.assertIn("conversation", GROUNDED_ANSWER_INSTRUCTIONS.lower())
|
||||||
|
self.assertNotIn("Answer ONLY from those sources", GROUNDED_ANSWER_INSTRUCTIONS)
|
||||||
|
|
||||||
|
def test_prior_turns_drops_newest_matching_user_text(self):
|
||||||
|
messages = [
|
||||||
|
HumanMessage(content="I don't know. Why?"),
|
||||||
|
AIMessage(content="the joke"),
|
||||||
|
HumanMessage(content="Tell me a joke"),
|
||||||
|
]
|
||||||
|
kept = prior_turns(messages, "I don't know. Why?")
|
||||||
|
self.assertEqual(
|
||||||
|
[message.content for message in kept],
|
||||||
|
["the joke", "Tell me a joke"],
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
class PromptBudgetTestCase(SimpleTestCase):
|
class PromptBudgetTestCase(SimpleTestCase):
|
||||||
def test_window_history_drops_oldest_first(self):
|
def test_window_history_drops_oldest_first(self):
|
||||||
messages = [HumanMessage(content="x" * 40) for _ in range(10)]
|
messages = [HumanMessage(content="x" * 40) for _ in range(10)]
|
||||||
@@ -99,7 +147,5 @@ class PromptBudgetTestCase(SimpleTestCase):
|
|||||||
self.assertEqual(kept[-1].content, messages[-1].content)
|
self.assertEqual(kept[-1].content, messages[-1].content)
|
||||||
|
|
||||||
def test_format_history_labels_speakers(self):
|
def test_format_history_labels_speakers(self):
|
||||||
text = format_history(
|
text = format_history([HumanMessage(content="hello"), AIMessage(content="hi")])
|
||||||
[HumanMessage(content="hello"), AIMessage(content="hi")]
|
|
||||||
)
|
|
||||||
self.assertEqual(text, "User: hello\nAI: hi")
|
self.assertEqual(text, "User: hello\nAI: hi")
|
||||||
@@ -1,4 +1,7 @@
|
|||||||
|
from datetime import timedelta
|
||||||
|
|
||||||
from django.urls import reverse
|
from django.urls import reverse
|
||||||
|
from django.utils import timezone
|
||||||
from rest_framework import status
|
from rest_framework import status
|
||||||
from rest_framework.test import APITestCase
|
from rest_framework.test import APITestCase
|
||||||
|
|
||||||
@@ -57,7 +60,7 @@ class ConversationPreferencesTestCase(APITestCase):
|
|||||||
|
|
||||||
self.assertEqual(response.status_code, status.HTTP_200_OK)
|
self.assertEqual(response.status_code, status.HTTP_200_OK)
|
||||||
self.assertTrue(response.data["order"])
|
self.assertTrue(response.data["order"])
|
||||||
self.assertFalse(response.data["use_conversation_context"])
|
self.assertTrue(response.data["use_conversation_context"])
|
||||||
|
|
||||||
def test_post_toggles_and_persists_order(self):
|
def test_post_toggles_and_persists_order(self):
|
||||||
response = self.client.post(self.url)
|
response = self.client.post(self.url)
|
||||||
@@ -115,8 +118,33 @@ class ConversationDetailViewTestCase(APITestCase):
|
|||||||
[("hello", True), ("hi there", False)],
|
[("hello", True), ("hi there", False)],
|
||||||
)
|
)
|
||||||
|
|
||||||
|
def test_get_returns_prompts_oldest_first_even_if_inserted_newest_first(self):
|
||||||
|
now = timezone.now()
|
||||||
|
make_prompt(
|
||||||
|
self.conversation,
|
||||||
|
message="later",
|
||||||
|
user_created=False,
|
||||||
|
created=now,
|
||||||
|
)
|
||||||
|
make_prompt(
|
||||||
|
self.conversation,
|
||||||
|
message="earlier",
|
||||||
|
user_created=True,
|
||||||
|
created=now - timedelta(minutes=5),
|
||||||
|
)
|
||||||
|
|
||||||
|
response = self.client.get(self.url, {"conversation_id": self.conversation.id})
|
||||||
|
|
||||||
|
self.assertEqual(response.status_code, status.HTTP_200_OK)
|
||||||
|
self.assertEqual(
|
||||||
|
[item["message"] for item in response.data],
|
||||||
|
["earlier", "later"],
|
||||||
|
)
|
||||||
|
|
||||||
def test_get_rejects_other_users_conversation(self):
|
def test_get_rejects_other_users_conversation(self):
|
||||||
other = make_user(email="other-tenant@example.com", company=make_company("Other"))
|
other = make_user(
|
||||||
|
email="other-tenant@example.com", company=make_company("Other")
|
||||||
|
)
|
||||||
foreign = make_conversation(user=other)
|
foreign = make_conversation(user=other)
|
||||||
make_prompt(foreign, message="secret")
|
make_prompt(foreign, message="secret")
|
||||||
|
|
||||||
|
|||||||
@@ -680,7 +680,7 @@ class ConversationDetailView(APIView):
|
|||||||
prompts = list(
|
prompts = list(
|
||||||
Prompt.objects.filter(
|
Prompt.objects.filter(
|
||||||
conversation__id=conversation_id, conversation__user=request.user
|
conversation__id=conversation_id, conversation__user=request.user
|
||||||
)
|
).order_by("created", "id")
|
||||||
)
|
)
|
||||||
serializer = PromptSerializer(
|
serializer = PromptSerializer(
|
||||||
prompts,
|
prompts,
|
||||||
@@ -734,7 +734,7 @@ class ConversationDetailView(APIView):
|
|||||||
messages = []
|
messages = []
|
||||||
for prompt_obj in Prompt.objects.filter(
|
for prompt_obj in Prompt.objects.filter(
|
||||||
conversation__id=conversation_id
|
conversation__id=conversation_id
|
||||||
):
|
).order_by("created", "id"):
|
||||||
messages.append(
|
messages.append(
|
||||||
{
|
{
|
||||||
"content": prompt_obj.message,
|
"content": prompt_obj.message,
|
||||||
|
|||||||
Reference in new issue
Block a user