From 9a07497b3810f1186d950442ba6eb545b4d7e8db Mon Sep 17 00:00:00 2001 From: Ryan Westfall Date: Tue, 4 Aug 2026 05:03:39 -0500 Subject: [PATCH] Add PromptFeedback API for per-message thumbs ratings (#67). Upsert/delete prompt_feedback and hydrate caller ratings on conversation_details so chat_web_app message actions can persist votes. --- llm_be/chat_backend/admin.py | 10 ++ .../migrations/0032_promptfeedback.py | 87 ++++++++++ llm_be/chat_backend/models.py | 53 +++++++ llm_be/chat_backend/serializers.py | 75 ++++++++- .../tests/test_views_prompt_feedback.py | 149 ++++++++++++++++++ llm_be/chat_backend/urls.py | 6 + llm_be/chat_backend/views.py | 104 +++++++++++- 7 files changed, 479 insertions(+), 5 deletions(-) create mode 100644 llm_be/chat_backend/migrations/0032_promptfeedback.py create mode 100644 llm_be/chat_backend/tests/test_views_prompt_feedback.py diff --git a/llm_be/chat_backend/admin.py b/llm_be/chat_backend/admin.py index 3a8852c..6aeffb7 100644 --- a/llm_be/chat_backend/admin.py +++ b/llm_be/chat_backend/admin.py @@ -8,6 +8,7 @@ from .models import ( Conversation, Prompt, Feedback, + PromptFeedback, PromptMetric, DocumentWorkspace, Document, @@ -134,6 +135,14 @@ class FeedbackAdmin(admin.ModelAdmin): list_display = ("status", "get_user_email", "title", "category") +class PromptFeedbackAdmin(admin.ModelAdmin): + model = PromptFeedback + list_display = ("id", "prompt", "user", "rating", "reason", "created") + list_filter = ("rating", "reason") + search_fields = ("user__email", "comment", "prompt__message") + raw_id_fields = ("prompt", "user") + + class LLMModelsAdmin(admin.ModelAdmin): model = LLMModels list_display = ("name", "port", "description") @@ -259,6 +268,7 @@ admin.site.register(Conversation, ConversationAdmin) admin.site.register(Prompt, PromptAdmin) admin.site.register(PromptMetric, PromptMetricAdmin) admin.site.register(Feedback, FeedbackAdmin) +admin.site.register(PromptFeedback, PromptFeedbackAdmin) admin.site.register(DocumentWorkspace, DocumentWorkspaceAdmin) admin.site.register(Document, DocumentAdmin) diff --git a/llm_be/chat_backend/migrations/0032_promptfeedback.py b/llm_be/chat_backend/migrations/0032_promptfeedback.py new file mode 100644 index 0000000..fd9554a --- /dev/null +++ b/llm_be/chat_backend/migrations/0032_promptfeedback.py @@ -0,0 +1,87 @@ +# Generated manually for chat_backend#67 + +import django.db.models.deletion +import django.utils.timezone +from django.conf import settings +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("chat_backend", "0031_prompt_citations"), + ] + + operations = [ + migrations.CreateModel( + name="PromptFeedback", + fields=[ + ( + "id", + models.BigAutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ("created", models.DateTimeField(default=django.utils.timezone.now)), + ( + "last_modified", + models.DateTimeField(default=django.utils.timezone.now), + ), + ( + "rating", + models.CharField( + choices=[("up", "Up"), ("down", "Down")], max_length=8 + ), + ), + ( + "reason", + models.CharField( + blank=True, + choices=[ + ("incorrect", "Incorrect"), + ("out_of_date", "Out of date"), + ( + "didnt_follow_instructions", + "Didn't follow instructions", + ), + ("unsafe", "Unsafe"), + ("other", "Other"), + ], + max_length=64, + null=True, + ), + ), + ( + "comment", + models.TextField(blank=True, max_length=1024, null=True), + ), + ( + "prompt", + models.ForeignKey( + help_text="Assistant prompt being rated", + on_delete=django.db.models.deletion.CASCADE, + related_name="prompt_feedbacks", + to="chat_backend.prompt", + ), + ), + ( + "user", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + related_name="prompt_feedbacks", + to=settings.AUTH_USER_MODEL, + ), + ), + ], + ), + migrations.AddConstraint( + model_name="promptfeedback", + constraint=models.UniqueConstraint( + fields=("prompt", "user"), + name="uniq_prompt_feedback_prompt_user", + ), + ), + ] diff --git a/llm_be/chat_backend/models.py b/llm_be/chat_backend/models.py index 2b88c3f..7f346f6 100644 --- a/llm_be/chat_backend/models.py +++ b/llm_be/chat_backend/models.py @@ -329,6 +329,59 @@ class Prompt(TimeInfoBase): return self.file != None and self.file.storage.exists(self.file.name) +class PromptFeedback(TimeInfoBase): + """Per-message thumbs rating for an assistant Prompt (chat_backend#67). + + Distinct from app-wide ``Feedback`` (product bugs). Joinable to + ``PromptMetric`` via ``prompt_id`` for per-model accuracy slices. + """ + + class Rating(models.TextChoices): + UP = "up", "Up" + DOWN = "down", "Down" + + class Reason(models.TextChoices): + INCORRECT = "incorrect", "Incorrect" + OUT_OF_DATE = "out_of_date", "Out of date" + DIDNT_FOLLOW_INSTRUCTIONS = ( + "didnt_follow_instructions", + "Didn't follow instructions", + ) + UNSAFE = "unsafe", "Unsafe" + OTHER = "other", "Other" + + prompt = models.ForeignKey( + Prompt, + on_delete=models.CASCADE, + related_name="prompt_feedbacks", + help_text="Assistant prompt being rated", + ) + user = models.ForeignKey( + CustomUser, + on_delete=models.CASCADE, + related_name="prompt_feedbacks", + ) + rating = models.CharField(max_length=8, choices=Rating.choices) + reason = models.CharField( + max_length=64, + choices=Reason.choices, + blank=True, + null=True, + ) + comment = models.TextField(max_length=1024, blank=True, null=True) + + class Meta: + constraints = [ + models.UniqueConstraint( + fields=("prompt", "user"), + name="uniq_prompt_feedback_prompt_user", + ) + ] + + def __str__(self): + return f"PromptFeedback(prompt={self.prompt_id}, user={self.user_id}, {self.rating})" + + class PromptMetric(TimeInfoBase): PROMPT_METRIC_CHOICES = ( ("CREATED", "Created"), diff --git a/llm_be/chat_backend/serializers.py b/llm_be/chat_backend/serializers.py index f162244..4e364ee 100644 --- a/llm_be/chat_backend/serializers.py +++ b/llm_be/chat_backend/serializers.py @@ -8,6 +8,7 @@ from .models import ( Company, Conversation, Prompt, + PromptFeedback, PromptMetric, Feedback, FEEDBACK_CATEGORIES, @@ -210,9 +211,52 @@ class ConversationSerializer(serializers.ModelSerializer): return tout +class PromptFeedbackSerializer(serializers.ModelSerializer): + prompt_id = serializers.IntegerField(source="prompt.id", read_only=True) + + class Meta: + model = PromptFeedback + fields = ( + "id", + "prompt_id", + "rating", + "reason", + "comment", + "created", + "last_modified", + ) + read_only_fields = ("id", "prompt_id", "created", "last_modified") + + +class PromptFeedbackUpsertSerializer(serializers.Serializer): + prompt_id = serializers.IntegerField() + rating = serializers.ChoiceField(choices=PromptFeedback.Rating.choices) + reason = serializers.ChoiceField( + choices=PromptFeedback.Reason.choices, + required=False, + allow_null=True, + allow_blank=True, + ) + comment = serializers.CharField( + required=False, allow_null=True, allow_blank=True, max_length=1024 + ) + + def validate_reason(self, value): + if value == "": + return None + return value + + def validate_comment(self, value): + if value is None: + return None + stripped = str(value).strip() + return stripped or None + + class PromptSerializer(serializers.ModelSerializer): tokens_in = serializers.SerializerMethodField() tokens_out = serializers.SerializerMethodField() + feedback = serializers.SerializerMethodField() class Meta: model = Prompt @@ -224,8 +268,9 @@ class PromptSerializer(serializers.ModelSerializer): "tokens_in", "tokens_out", "citations", + "feedback", ) - read_only_fields = ("citations",) + read_only_fields = ("citations", "feedback") def _token_pair(self, obj): cache = self.context.setdefault("_prompt_token_cache", {}) @@ -241,6 +286,34 @@ class PromptSerializer(serializers.ModelSerializer): _, tout = self._token_pair(obj) return tout + def get_feedback(self, obj): + """Current caller's rating for this prompt, if any.""" + request = self.context.get("request") + if request is None or not getattr(request, "user", None): + return None + user = request.user + if not user.is_authenticated: + return None + + by_prompt = self.context.get("_prompt_feedback_by_id") + if by_prompt is None: + prompt_ids = self.context.get("_prompt_ids_for_feedback") + qs = PromptFeedback.objects.filter(user=user).only( + "prompt_id", "rating", "reason", "comment" + ) + if prompt_ids is not None: + qs = qs.filter(prompt_id__in=prompt_ids) + by_prompt = { + row.prompt_id: { + "rating": row.rating, + "reason": row.reason, + "comment": row.comment, + } + for row in qs + } + self.context["_prompt_feedback_by_id"] = by_prompt + return by_prompt.get(obj.id) + def validate_message(self, value: str) -> str: if value is None or not str(value).strip(): raise serializers.ValidationError("Message text cannot be empty.") diff --git a/llm_be/chat_backend/tests/test_views_prompt_feedback.py b/llm_be/chat_backend/tests/test_views_prompt_feedback.py new file mode 100644 index 0000000..3cbc446 --- /dev/null +++ b/llm_be/chat_backend/tests/test_views_prompt_feedback.py @@ -0,0 +1,149 @@ +from django.urls import reverse +from rest_framework import status +from rest_framework.test import APITestCase + +from chat_backend.models import PromptFeedback, PromptMetric + +from .factories import make_company, make_conversation, make_prompt, make_user + + +class PromptFeedbackViewTestCase(APITestCase): + def setUp(self): + self.user = make_user(company=make_company()) + self.client.force_authenticate(user=self.user) + self.conversation = make_conversation(user=self.user) + self.assistant = make_prompt( + self.conversation, message="answer", user_created=False + ) + self.user_prompt = make_prompt( + self.conversation, message="question", user_created=True + ) + self.url = reverse("prompt_feedback") + self.details_url = reverse("conversation_details") + + def test_upsert_creates_unique_row(self): + response = self.client.post( + self.url, + {"prompt_id": self.assistant.id, "rating": "up"}, + format="json", + ) + + self.assertEqual(response.status_code, status.HTTP_200_OK) + self.assertEqual(response.data["rating"], "up") + self.assertEqual(response.data["prompt_id"], self.assistant.id) + self.assertEqual(PromptFeedback.objects.count(), 1) + + again = self.client.post( + self.url, + { + "prompt_id": self.assistant.id, + "rating": "down", + "reason": "incorrect", + "comment": "wrong cite", + }, + format="json", + ) + + self.assertEqual(again.status_code, status.HTTP_200_OK) + self.assertEqual(PromptFeedback.objects.count(), 1) + row = PromptFeedback.objects.get() + self.assertEqual(row.rating, "down") + self.assertEqual(row.reason, "incorrect") + self.assertEqual(row.comment, "wrong cite") + + def test_delete_clears_vote(self): + PromptFeedback.objects.create( + prompt=self.assistant, user=self.user, rating="up" + ) + + response = self.client.delete( + f"{self.url}?prompt_id={self.assistant.id}" + ) + + self.assertEqual(response.status_code, status.HTTP_204_NO_CONTENT) + self.assertEqual(PromptFeedback.objects.count(), 0) + + def test_delete_missing_vote_is_404(self): + response = self.client.delete( + f"{self.url}?prompt_id={self.assistant.id}" + ) + self.assertEqual(response.status_code, status.HTTP_404_NOT_FOUND) + + def test_cannot_rate_user_prompt(self): + response = self.client.post( + self.url, + {"prompt_id": self.user_prompt.id, "rating": "up"}, + format="json", + ) + self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST) + self.assertEqual(PromptFeedback.objects.count(), 0) + + def test_cannot_rate_other_users_prompt(self): + other = make_user(email="other@example.com", company=make_company("O")) + foreign = make_prompt( + make_conversation(user=other), message="secret", user_created=False + ) + + response = self.client.post( + self.url, + {"prompt_id": foreign.id, "rating": "up"}, + format="json", + ) + self.assertEqual(response.status_code, status.HTTP_404_NOT_FOUND) + self.assertEqual(PromptFeedback.objects.count(), 0) + + def test_conversation_details_includes_caller_feedback(self): + PromptFeedback.objects.create( + prompt=self.assistant, + user=self.user, + rating="down", + reason="unsafe", + comment="bad", + ) + # Another user's vote must not leak + other = make_user(email="peer@example.com", company=self.user.company) + PromptFeedback.objects.create( + prompt=self.assistant, user=other, rating="up" + ) + + response = self.client.get( + self.details_url, {"conversation_id": self.conversation.id} + ) + + self.assertEqual(response.status_code, status.HTTP_200_OK) + by_id = {item["id"]: item for item in response.data} + self.assertIsNone(by_id[self.user_prompt.id]["feedback"]) + self.assertEqual( + by_id[self.assistant.id]["feedback"], + {"rating": "down", "reason": "unsafe", "comment": "bad"}, + ) + + def test_feedback_joinable_to_prompt_metric(self): + PromptFeedback.objects.create( + prompt=self.assistant, user=self.user, rating="up" + ) + PromptMetric.objects.create( + prompt_id=self.assistant.id, + conversation_id=self.conversation.id, + event="FINISHED", + model_name="llama3.2", + start_time=self.assistant.created, + prompt_length=10, + has_file=False, + ) + + joined = PromptFeedback.objects.filter( + prompt_id__in=PromptMetric.objects.filter( + model_name="llama3.2" + ).values_list("prompt_id", flat=True) + ) + self.assertEqual(joined.count(), 1) + + def test_unauthenticated_rejected(self): + self.client.force_authenticate(user=None) + response = self.client.post( + self.url, + {"prompt_id": self.assistant.id, "rating": "up"}, + format="json", + ) + self.assertEqual(response.status_code, status.HTTP_401_UNAUTHORIZED) diff --git a/llm_be/chat_backend/urls.py b/llm_be/chat_backend/urls.py index 27412b8..81bb8e2 100644 --- a/llm_be/chat_backend/urls.py +++ b/llm_be/chat_backend/urls.py @@ -12,6 +12,7 @@ from .views import ( is_authenticated, AnnouncmentView, FeedbackView, + PromptFeedbackView, ConversationsView, ConversationDetailView, CompanyUsersView, @@ -78,6 +79,11 @@ urlpatterns = [ path("announcment/get/", AnnouncmentView.as_view(), name="get_announcments"), path("conversations", ConversationsView.as_view(), name="conversations"), path("feedbacks/", FeedbackView.as_view(), name="feedbacks"), + path( + "prompt_feedback", + PromptFeedbackView.as_view(), + name="prompt_feedback", + ), path( "conversation_details", ConversationDetailView.as_view(), diff --git a/llm_be/chat_backend/views.py b/llm_be/chat_backend/views.py index 8c2326f..678a0b2 100644 --- a/llm_be/chat_backend/views.py +++ b/llm_be/chat_backend/views.py @@ -13,6 +13,8 @@ from .serializers import ( ConversationSerializer, PromptSerializer, FeedbackSerializer, + PromptFeedbackSerializer, + PromptFeedbackUpsertSerializer, DocumentWorkspaceSerializer, DocumentSerializer, ) @@ -24,6 +26,7 @@ from .models import ( Announcement, Conversation, Prompt, + PromptFeedback, Feedback, PromptMetric, DocumentWorkspace, @@ -380,6 +383,90 @@ class FeedbackView(APIView): return Response(serializer.data, status=status.HTTP_200_OK) +def _user_can_rate_prompt(user, prompt: Prompt) -> bool: + """Caller may rate prompts in their own non-deleted conversations.""" + conversation = prompt.conversation + return ( + conversation is not None + and conversation.user_id == user.id + and not conversation.deleted + ) + + +class PromptFeedbackView(APIView): + """Upsert / clear per-message thumbs ratings (chat_backend#67).""" + + http_method_names = ["post", "delete"] + + def post(self, request, format="json"): + serializer = PromptFeedbackUpsertSerializer(data=request.data) + if not serializer.is_valid(): + return Response(serializer.errors, status=status.HTTP_400_BAD_REQUEST) + + prompt_id = serializer.validated_data["prompt_id"] + try: + prompt = Prompt.objects.select_related("conversation").get(id=prompt_id) + except Prompt.DoesNotExist: + return Response( + {"detail": "Prompt not found."}, + status=status.HTTP_404_NOT_FOUND, + ) + + if not _user_can_rate_prompt(request.user, prompt): + return Response( + {"detail": "Prompt not found."}, + status=status.HTTP_404_NOT_FOUND, + ) + + if prompt.user_created: + return Response( + {"detail": "Only assistant prompts can be rated."}, + status=status.HTTP_400_BAD_REQUEST, + ) + + feedback, _created = PromptFeedback.objects.update_or_create( + prompt=prompt, + user=request.user, + defaults={ + "rating": serializer.validated_data["rating"], + "reason": serializer.validated_data.get("reason"), + "comment": serializer.validated_data.get("comment"), + }, + ) + return Response( + PromptFeedbackSerializer(feedback).data, + status=status.HTTP_200_OK, + ) + + def delete(self, request, format="json"): + prompt_id = request.query_params.get("prompt_id") + if prompt_id is None: + return Response( + {"detail": "prompt_id is required."}, + status=status.HTTP_400_BAD_REQUEST, + ) + try: + prompt_id = int(prompt_id) + except (TypeError, ValueError): + return Response( + {"detail": "prompt_id must be an integer."}, + status=status.HTTP_400_BAD_REQUEST, + ) + + deleted, _ = PromptFeedback.objects.filter( + prompt_id=prompt_id, + user=request.user, + prompt__conversation__user=request.user, + prompt__conversation__deleted=False, + ).delete() + if not deleted: + return Response( + {"detail": "Prompt feedback not found."}, + status=status.HTTP_404_NOT_FOUND, + ) + return Response(status=status.HTTP_204_NO_CONTENT) + + class AcknowledgeTermsOfService(APIView): http_method_names = ["post"] @@ -516,11 +603,20 @@ class ConversationDetailView(APIView): {"detail": "Conversation not found."}, status=status.HTTP_404_NOT_FOUND, ) - prompts = Prompt.objects.filter( - conversation__id=conversation_id, conversation__user=request.user + prompts = list( + Prompt.objects.filter( + conversation__id=conversation_id, conversation__user=request.user + ) ) - serailzer = PromptSerializer(prompts, many=True) - return Response(serailzer.data, status=status.HTTP_200_OK) + serializer = PromptSerializer( + prompts, + many=True, + context={ + "request": request, + "_prompt_ids_for_feedback": [p.id for p in prompts], + }, + ) + return Response(serializer.data, status=status.HTTP_200_OK) def post(self, request, format="json"): logger.info("In the post") -- 2.54.0