## Summary - Closes Phase 4 of [#62](#62): `evals/suite.json` (≥40 graded questions), `run_evals` management command, and manually-triggered `.gitea/workflows/run-evals.yml`. - Emits versioned WS `status` frames during grounded chat (evaluating / searching / reading_sources / refining / writing) for [chat_web_app#96](ai_ml_operations/chat_web_app#96). - Implements [#63](#63): Redis/Celery optional infra, `AgentRun`/`AgentStep`, tool registry (SSRF-safe `fetch_url`, tenant-scoped docs), LangGraph orchestrator, progress frames, REST `GET/POST /api/agent_runs/…`, gated by `ALLOW_AGENTIC_TASKS` (default off). ## Test plan - [x] `SKIP_RAG_INIT=1 uv run python manage.py test` for evals, ws frames, agent tools, consumers, grounding - [ ] Manual: with `ALLOW_AGENTIC_TASKS=false`, chat identical to today - [ ] Manual: status frames visible in FE with #96 branch - [ ] Manual (GPU): `python manage.py run_evals --runs 3` - [ ] Manual: `ALLOW_AGENTIC_TASKS=true` multi-step research prompt creates AgentRun + framesReviewed-on: #71
This commit was merged in pull request #71.
This commit is contained in:
westfarn committed
1 parent
e1e086a474
commit
093e5462a5
46 files changed
+4669
-117
No files matched your search
@@ -29,7 +29,13 @@ from .services.title_generator import title_generator
|
||||
from .services.moderation_classifier import moderation_classifier, ModerationLabel
|
||||
from .services.prompt_classifier.prompt_classifier import PromptClassifier, PromptType
|
||||
from .services.data_analysis_service import AsyncDataAnalysisService
|
||||
from .services.grounded_chat import citations_frame, prepare_grounded_chat
|
||||
from .services.grounded_chat import prepare_grounded_chat
|
||||
from .services.status_context import (
|
||||
emit_status,
|
||||
reset_status_emitter,
|
||||
set_status_emitter,
|
||||
)
|
||||
from .services.ws_frames import citations_frame, status_frame
|
||||
from chat_backend.ollama_config import ollama_model_for_role, resolve_chat_role
|
||||
from .utils import (
|
||||
TokenUsageCollector,
|
||||
@@ -230,6 +236,7 @@ class ChatState(TypedDict):
|
||||
# --- LangGraph Nodes ---
|
||||
|
||||
async def moderation_node(state: ChatState) -> ChatState:
|
||||
await emit_status("moderating")
|
||||
msg = state["message"]
|
||||
label = await moderation_classifier.classify_async(msg)
|
||||
return {"moderation_label": label}
|
||||
@@ -297,13 +304,16 @@ async def generation_node(state: ChatState) -> ChatState:
|
||||
}
|
||||
service = AsyncRAGService()
|
||||
workspace = await get_workspace(conversation_id, user=chat_user)
|
||||
await emit_status("retrieving_docs")
|
||||
generator = service.generate_response(messages, prompt_instance.message, workspace)
|
||||
await emit_status("refining")
|
||||
return {"response_generator": generator}
|
||||
|
||||
elif prompt_type == PromptType.DATA_ANALYSIS:
|
||||
service = AsyncDataAnalysisService()
|
||||
if not decoded_file:
|
||||
return {"response_generator": {"type": "text", "content": "Please upload a file to perform data analysis."}}
|
||||
await emit_status("analysing")
|
||||
generator = service.generate_response(prompt_instance.message, decoded_file, file_type)
|
||||
return {"response_generator": generator}
|
||||
|
||||
@@ -509,58 +519,71 @@ class ChatConsumerGraph(AsyncWebsocketConsumer):
|
||||
}
|
||||
print("Initial State: ", initial_state)
|
||||
|
||||
# Run Graph
|
||||
final_state = await app.ainvoke(initial_state)
|
||||
print("Final State: ", final_state)
|
||||
|
||||
response_generator_or_dict = final_state["response_generator"]
|
||||
print("Response Generator: ", response_generator_or_dict)
|
||||
|
||||
# Send start markers
|
||||
# Stream markers early so status frames reach the client (#96).
|
||||
await self.send("CONVERSATION_ID")
|
||||
await self.send(str(conversation_id))
|
||||
await self.send("START_OF_THE_STREAM_ENDER_GAME_42")
|
||||
|
||||
full_response = ""
|
||||
tokens_in = tokens_out = None
|
||||
|
||||
if isinstance(response_generator_or_dict, dict):
|
||||
content = response_generator_or_dict.get("content", "")
|
||||
await self.send_json_message(json.dumps(response_generator_or_dict))
|
||||
full_response = content
|
||||
tokens_in, tokens_out = extract_token_usage(
|
||||
response_generator_or_dict
|
||||
)
|
||||
else:
|
||||
# Stream raw LLM chunks so final Ollama generation_info
|
||||
# (prompt_eval_count / eval_count) is not stripped.
|
||||
usage = TokenUsageCollector()
|
||||
async for chunk in aiter_text_chunks(
|
||||
response_generator_or_dict, usage
|
||||
):
|
||||
full_response += chunk
|
||||
await self.send_json_message(chunk)
|
||||
tokens_in, tokens_out = usage.pair
|
||||
|
||||
await self.send("END_OF_THE_STREAM_ENDER_GAME_42")
|
||||
|
||||
citations = final_state.get("citations") or []
|
||||
if citations:
|
||||
await self.send_json_message(json.dumps(citations_frame(citations)))
|
||||
|
||||
final_model = final_state.get("resolved_model") or resolved_model
|
||||
if final_model and final_model != prompt_metric.model_name:
|
||||
prompt_metric.model_name = final_model
|
||||
await database_sync_to_async(prompt_metric.save)(
|
||||
update_fields=["model_name"]
|
||||
async def _send_status(stage, detail=None):
|
||||
await self.send_json_message(
|
||||
json.dumps(status_frame(stage, detail=detail))
|
||||
)
|
||||
|
||||
await save_generated_message(
|
||||
conversation_id, full_response, citations=citations
|
||||
)
|
||||
await finish_prompt_metric(
|
||||
prompt_metric,
|
||||
len(full_response),
|
||||
tokens_in=tokens_in,
|
||||
tokens_out=tokens_out,
|
||||
)
|
||||
status_token = set_status_emitter(_send_status)
|
||||
try:
|
||||
await emit_status("queued")
|
||||
# Run Graph (moderation emits moderating; grounding emits evaluating/…)
|
||||
final_state = await app.ainvoke(initial_state)
|
||||
print("Final State: ", final_state)
|
||||
|
||||
response_generator_or_dict = final_state["response_generator"]
|
||||
print("Response Generator: ", response_generator_or_dict)
|
||||
|
||||
full_response = ""
|
||||
tokens_in = tokens_out = None
|
||||
|
||||
if isinstance(response_generator_or_dict, dict):
|
||||
content = response_generator_or_dict.get("content", "")
|
||||
await self.send_json_message(
|
||||
json.dumps(response_generator_or_dict)
|
||||
)
|
||||
full_response = content
|
||||
tokens_in, tokens_out = extract_token_usage(
|
||||
response_generator_or_dict
|
||||
)
|
||||
else:
|
||||
await emit_status("writing")
|
||||
usage = TokenUsageCollector()
|
||||
async for chunk in aiter_text_chunks(
|
||||
response_generator_or_dict, usage
|
||||
):
|
||||
full_response += chunk
|
||||
await self.send_json_message(chunk)
|
||||
tokens_in, tokens_out = usage.pair
|
||||
|
||||
await self.send("END_OF_THE_STREAM_ENDER_GAME_42")
|
||||
|
||||
citations = final_state.get("citations") or []
|
||||
if citations:
|
||||
await self.send_json_message(
|
||||
json.dumps(citations_frame(citations))
|
||||
)
|
||||
|
||||
final_model = final_state.get("resolved_model") or resolved_model
|
||||
if final_model and final_model != prompt_metric.model_name:
|
||||
prompt_metric.model_name = final_model
|
||||
await database_sync_to_async(prompt_metric.save)(
|
||||
update_fields=["model_name"]
|
||||
)
|
||||
|
||||
await save_generated_message(
|
||||
conversation_id, full_response, citations=citations
|
||||
)
|
||||
await finish_prompt_metric(
|
||||
prompt_metric,
|
||||
len(full_response),
|
||||
tokens_in=tokens_in,
|
||||
tokens_out=tokens_out,
|
||||
)
|
||||
finally:
|
||||
reset_status_emitter(status_token)
|
||||
Reference in new issue
Block a user