Compare commits
1
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
07c03ce65d |
@@ -8,7 +8,6 @@ from .models import (
|
||||
Conversation,
|
||||
Prompt,
|
||||
Feedback,
|
||||
PromptFeedback,
|
||||
PromptMetric,
|
||||
DocumentWorkspace,
|
||||
Document,
|
||||
@@ -135,14 +134,6 @@ class FeedbackAdmin(admin.ModelAdmin):
|
||||
list_display = ("status", "get_user_email", "title", "category")
|
||||
|
||||
|
||||
class PromptFeedbackAdmin(admin.ModelAdmin):
|
||||
model = PromptFeedback
|
||||
list_display = ("id", "prompt", "user", "rating", "reason", "created")
|
||||
list_filter = ("rating", "reason")
|
||||
search_fields = ("user__email", "comment", "prompt__message")
|
||||
raw_id_fields = ("prompt", "user")
|
||||
|
||||
|
||||
class LLMModelsAdmin(admin.ModelAdmin):
|
||||
model = LLMModels
|
||||
list_display = ("name", "port", "description")
|
||||
@@ -268,7 +259,6 @@ admin.site.register(Conversation, ConversationAdmin)
|
||||
admin.site.register(Prompt, PromptAdmin)
|
||||
admin.site.register(PromptMetric, PromptMetricAdmin)
|
||||
admin.site.register(Feedback, FeedbackAdmin)
|
||||
admin.site.register(PromptFeedback, PromptFeedbackAdmin)
|
||||
|
||||
admin.site.register(DocumentWorkspace, DocumentWorkspaceAdmin)
|
||||
admin.site.register(Document, DocumentAdmin)
|
||||
|
||||
@@ -1,87 +0,0 @@
|
||||
# Generated manually for chat_backend#67
|
||||
|
||||
import django.db.models.deletion
|
||||
import django.utils.timezone
|
||||
from django.conf import settings
|
||||
from django.db import migrations, models
|
||||
|
||||
|
||||
class Migration(migrations.Migration):
|
||||
|
||||
dependencies = [
|
||||
("chat_backend", "0031_prompt_citations"),
|
||||
]
|
||||
|
||||
operations = [
|
||||
migrations.CreateModel(
|
||||
name="PromptFeedback",
|
||||
fields=[
|
||||
(
|
||||
"id",
|
||||
models.BigAutoField(
|
||||
auto_created=True,
|
||||
primary_key=True,
|
||||
serialize=False,
|
||||
verbose_name="ID",
|
||||
),
|
||||
),
|
||||
("created", models.DateTimeField(default=django.utils.timezone.now)),
|
||||
(
|
||||
"last_modified",
|
||||
models.DateTimeField(default=django.utils.timezone.now),
|
||||
),
|
||||
(
|
||||
"rating",
|
||||
models.CharField(
|
||||
choices=[("up", "Up"), ("down", "Down")], max_length=8
|
||||
),
|
||||
),
|
||||
(
|
||||
"reason",
|
||||
models.CharField(
|
||||
blank=True,
|
||||
choices=[
|
||||
("incorrect", "Incorrect"),
|
||||
("out_of_date", "Out of date"),
|
||||
(
|
||||
"didnt_follow_instructions",
|
||||
"Didn't follow instructions",
|
||||
),
|
||||
("unsafe", "Unsafe"),
|
||||
("other", "Other"),
|
||||
],
|
||||
max_length=64,
|
||||
null=True,
|
||||
),
|
||||
),
|
||||
(
|
||||
"comment",
|
||||
models.TextField(blank=True, max_length=1024, null=True),
|
||||
),
|
||||
(
|
||||
"prompt",
|
||||
models.ForeignKey(
|
||||
help_text="Assistant prompt being rated",
|
||||
on_delete=django.db.models.deletion.CASCADE,
|
||||
related_name="prompt_feedbacks",
|
||||
to="chat_backend.prompt",
|
||||
),
|
||||
),
|
||||
(
|
||||
"user",
|
||||
models.ForeignKey(
|
||||
on_delete=django.db.models.deletion.CASCADE,
|
||||
related_name="prompt_feedbacks",
|
||||
to=settings.AUTH_USER_MODEL,
|
||||
),
|
||||
),
|
||||
],
|
||||
),
|
||||
migrations.AddConstraint(
|
||||
model_name="promptfeedback",
|
||||
constraint=models.UniqueConstraint(
|
||||
fields=("prompt", "user"),
|
||||
name="uniq_prompt_feedback_prompt_user",
|
||||
),
|
||||
),
|
||||
]
|
||||
@@ -329,59 +329,6 @@ class Prompt(TimeInfoBase):
|
||||
return self.file != None and self.file.storage.exists(self.file.name)
|
||||
|
||||
|
||||
class PromptFeedback(TimeInfoBase):
|
||||
"""Per-message thumbs rating for an assistant Prompt (chat_backend#67).
|
||||
|
||||
Distinct from app-wide ``Feedback`` (product bugs). Joinable to
|
||||
``PromptMetric`` via ``prompt_id`` for per-model accuracy slices.
|
||||
"""
|
||||
|
||||
class Rating(models.TextChoices):
|
||||
UP = "up", "Up"
|
||||
DOWN = "down", "Down"
|
||||
|
||||
class Reason(models.TextChoices):
|
||||
INCORRECT = "incorrect", "Incorrect"
|
||||
OUT_OF_DATE = "out_of_date", "Out of date"
|
||||
DIDNT_FOLLOW_INSTRUCTIONS = (
|
||||
"didnt_follow_instructions",
|
||||
"Didn't follow instructions",
|
||||
)
|
||||
UNSAFE = "unsafe", "Unsafe"
|
||||
OTHER = "other", "Other"
|
||||
|
||||
prompt = models.ForeignKey(
|
||||
Prompt,
|
||||
on_delete=models.CASCADE,
|
||||
related_name="prompt_feedbacks",
|
||||
help_text="Assistant prompt being rated",
|
||||
)
|
||||
user = models.ForeignKey(
|
||||
CustomUser,
|
||||
on_delete=models.CASCADE,
|
||||
related_name="prompt_feedbacks",
|
||||
)
|
||||
rating = models.CharField(max_length=8, choices=Rating.choices)
|
||||
reason = models.CharField(
|
||||
max_length=64,
|
||||
choices=Reason.choices,
|
||||
blank=True,
|
||||
null=True,
|
||||
)
|
||||
comment = models.TextField(max_length=1024, blank=True, null=True)
|
||||
|
||||
class Meta:
|
||||
constraints = [
|
||||
models.UniqueConstraint(
|
||||
fields=("prompt", "user"),
|
||||
name="uniq_prompt_feedback_prompt_user",
|
||||
)
|
||||
]
|
||||
|
||||
def __str__(self):
|
||||
return f"PromptFeedback(prompt={self.prompt_id}, user={self.user_id}, {self.rating})"
|
||||
|
||||
|
||||
class PromptMetric(TimeInfoBase):
|
||||
PROMPT_METRIC_CHOICES = (
|
||||
("CREATED", "Created"),
|
||||
|
||||
@@ -8,7 +8,6 @@ from .models import (
|
||||
Company,
|
||||
Conversation,
|
||||
Prompt,
|
||||
PromptFeedback,
|
||||
PromptMetric,
|
||||
Feedback,
|
||||
FEEDBACK_CATEGORIES,
|
||||
@@ -211,52 +210,9 @@ class ConversationSerializer(serializers.ModelSerializer):
|
||||
return tout
|
||||
|
||||
|
||||
class PromptFeedbackSerializer(serializers.ModelSerializer):
|
||||
prompt_id = serializers.IntegerField(source="prompt.id", read_only=True)
|
||||
|
||||
class Meta:
|
||||
model = PromptFeedback
|
||||
fields = (
|
||||
"id",
|
||||
"prompt_id",
|
||||
"rating",
|
||||
"reason",
|
||||
"comment",
|
||||
"created",
|
||||
"last_modified",
|
||||
)
|
||||
read_only_fields = ("id", "prompt_id", "created", "last_modified")
|
||||
|
||||
|
||||
class PromptFeedbackUpsertSerializer(serializers.Serializer):
|
||||
prompt_id = serializers.IntegerField()
|
||||
rating = serializers.ChoiceField(choices=PromptFeedback.Rating.choices)
|
||||
reason = serializers.ChoiceField(
|
||||
choices=PromptFeedback.Reason.choices,
|
||||
required=False,
|
||||
allow_null=True,
|
||||
allow_blank=True,
|
||||
)
|
||||
comment = serializers.CharField(
|
||||
required=False, allow_null=True, allow_blank=True, max_length=1024
|
||||
)
|
||||
|
||||
def validate_reason(self, value):
|
||||
if value == "":
|
||||
return None
|
||||
return value
|
||||
|
||||
def validate_comment(self, value):
|
||||
if value is None:
|
||||
return None
|
||||
stripped = str(value).strip()
|
||||
return stripped or None
|
||||
|
||||
|
||||
class PromptSerializer(serializers.ModelSerializer):
|
||||
tokens_in = serializers.SerializerMethodField()
|
||||
tokens_out = serializers.SerializerMethodField()
|
||||
feedback = serializers.SerializerMethodField()
|
||||
|
||||
class Meta:
|
||||
model = Prompt
|
||||
@@ -268,9 +224,8 @@ class PromptSerializer(serializers.ModelSerializer):
|
||||
"tokens_in",
|
||||
"tokens_out",
|
||||
"citations",
|
||||
"feedback",
|
||||
)
|
||||
read_only_fields = ("citations", "feedback")
|
||||
read_only_fields = ("citations",)
|
||||
|
||||
def _token_pair(self, obj):
|
||||
cache = self.context.setdefault("_prompt_token_cache", {})
|
||||
@@ -286,34 +241,6 @@ class PromptSerializer(serializers.ModelSerializer):
|
||||
_, tout = self._token_pair(obj)
|
||||
return tout
|
||||
|
||||
def get_feedback(self, obj):
|
||||
"""Current caller's rating for this prompt, if any."""
|
||||
request = self.context.get("request")
|
||||
if request is None or not getattr(request, "user", None):
|
||||
return None
|
||||
user = request.user
|
||||
if not user.is_authenticated:
|
||||
return None
|
||||
|
||||
by_prompt = self.context.get("_prompt_feedback_by_id")
|
||||
if by_prompt is None:
|
||||
prompt_ids = self.context.get("_prompt_ids_for_feedback")
|
||||
qs = PromptFeedback.objects.filter(user=user).only(
|
||||
"prompt_id", "rating", "reason", "comment"
|
||||
)
|
||||
if prompt_ids is not None:
|
||||
qs = qs.filter(prompt_id__in=prompt_ids)
|
||||
by_prompt = {
|
||||
row.prompt_id: {
|
||||
"rating": row.rating,
|
||||
"reason": row.reason,
|
||||
"comment": row.comment,
|
||||
}
|
||||
for row in qs
|
||||
}
|
||||
self.context["_prompt_feedback_by_id"] = by_prompt
|
||||
return by_prompt.get(obj.id)
|
||||
|
||||
def validate_message(self, value: str) -> str:
|
||||
if value is None or not str(value).strip():
|
||||
raise serializers.ValidationError("Message text cannot be empty.")
|
||||
|
||||
@@ -1,149 +0,0 @@
|
||||
from django.urls import reverse
|
||||
from rest_framework import status
|
||||
from rest_framework.test import APITestCase
|
||||
|
||||
from chat_backend.models import PromptFeedback, PromptMetric
|
||||
|
||||
from .factories import make_company, make_conversation, make_prompt, make_user
|
||||
|
||||
|
||||
class PromptFeedbackViewTestCase(APITestCase):
|
||||
def setUp(self):
|
||||
self.user = make_user(company=make_company())
|
||||
self.client.force_authenticate(user=self.user)
|
||||
self.conversation = make_conversation(user=self.user)
|
||||
self.assistant = make_prompt(
|
||||
self.conversation, message="answer", user_created=False
|
||||
)
|
||||
self.user_prompt = make_prompt(
|
||||
self.conversation, message="question", user_created=True
|
||||
)
|
||||
self.url = reverse("prompt_feedback")
|
||||
self.details_url = reverse("conversation_details")
|
||||
|
||||
def test_upsert_creates_unique_row(self):
|
||||
response = self.client.post(
|
||||
self.url,
|
||||
{"prompt_id": self.assistant.id, "rating": "up"},
|
||||
format="json",
|
||||
)
|
||||
|
||||
self.assertEqual(response.status_code, status.HTTP_200_OK)
|
||||
self.assertEqual(response.data["rating"], "up")
|
||||
self.assertEqual(response.data["prompt_id"], self.assistant.id)
|
||||
self.assertEqual(PromptFeedback.objects.count(), 1)
|
||||
|
||||
again = self.client.post(
|
||||
self.url,
|
||||
{
|
||||
"prompt_id": self.assistant.id,
|
||||
"rating": "down",
|
||||
"reason": "incorrect",
|
||||
"comment": "wrong cite",
|
||||
},
|
||||
format="json",
|
||||
)
|
||||
|
||||
self.assertEqual(again.status_code, status.HTTP_200_OK)
|
||||
self.assertEqual(PromptFeedback.objects.count(), 1)
|
||||
row = PromptFeedback.objects.get()
|
||||
self.assertEqual(row.rating, "down")
|
||||
self.assertEqual(row.reason, "incorrect")
|
||||
self.assertEqual(row.comment, "wrong cite")
|
||||
|
||||
def test_delete_clears_vote(self):
|
||||
PromptFeedback.objects.create(
|
||||
prompt=self.assistant, user=self.user, rating="up"
|
||||
)
|
||||
|
||||
response = self.client.delete(
|
||||
f"{self.url}?prompt_id={self.assistant.id}"
|
||||
)
|
||||
|
||||
self.assertEqual(response.status_code, status.HTTP_204_NO_CONTENT)
|
||||
self.assertEqual(PromptFeedback.objects.count(), 0)
|
||||
|
||||
def test_delete_missing_vote_is_404(self):
|
||||
response = self.client.delete(
|
||||
f"{self.url}?prompt_id={self.assistant.id}"
|
||||
)
|
||||
self.assertEqual(response.status_code, status.HTTP_404_NOT_FOUND)
|
||||
|
||||
def test_cannot_rate_user_prompt(self):
|
||||
response = self.client.post(
|
||||
self.url,
|
||||
{"prompt_id": self.user_prompt.id, "rating": "up"},
|
||||
format="json",
|
||||
)
|
||||
self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST)
|
||||
self.assertEqual(PromptFeedback.objects.count(), 0)
|
||||
|
||||
def test_cannot_rate_other_users_prompt(self):
|
||||
other = make_user(email="other@example.com", company=make_company("O"))
|
||||
foreign = make_prompt(
|
||||
make_conversation(user=other), message="secret", user_created=False
|
||||
)
|
||||
|
||||
response = self.client.post(
|
||||
self.url,
|
||||
{"prompt_id": foreign.id, "rating": "up"},
|
||||
format="json",
|
||||
)
|
||||
self.assertEqual(response.status_code, status.HTTP_404_NOT_FOUND)
|
||||
self.assertEqual(PromptFeedback.objects.count(), 0)
|
||||
|
||||
def test_conversation_details_includes_caller_feedback(self):
|
||||
PromptFeedback.objects.create(
|
||||
prompt=self.assistant,
|
||||
user=self.user,
|
||||
rating="down",
|
||||
reason="unsafe",
|
||||
comment="bad",
|
||||
)
|
||||
# Another user's vote must not leak
|
||||
other = make_user(email="peer@example.com", company=self.user.company)
|
||||
PromptFeedback.objects.create(
|
||||
prompt=self.assistant, user=other, rating="up"
|
||||
)
|
||||
|
||||
response = self.client.get(
|
||||
self.details_url, {"conversation_id": self.conversation.id}
|
||||
)
|
||||
|
||||
self.assertEqual(response.status_code, status.HTTP_200_OK)
|
||||
by_id = {item["id"]: item for item in response.data}
|
||||
self.assertIsNone(by_id[self.user_prompt.id]["feedback"])
|
||||
self.assertEqual(
|
||||
by_id[self.assistant.id]["feedback"],
|
||||
{"rating": "down", "reason": "unsafe", "comment": "bad"},
|
||||
)
|
||||
|
||||
def test_feedback_joinable_to_prompt_metric(self):
|
||||
PromptFeedback.objects.create(
|
||||
prompt=self.assistant, user=self.user, rating="up"
|
||||
)
|
||||
PromptMetric.objects.create(
|
||||
prompt_id=self.assistant.id,
|
||||
conversation_id=self.conversation.id,
|
||||
event="FINISHED",
|
||||
model_name="llama3.2",
|
||||
start_time=self.assistant.created,
|
||||
prompt_length=10,
|
||||
has_file=False,
|
||||
)
|
||||
|
||||
joined = PromptFeedback.objects.filter(
|
||||
prompt_id__in=PromptMetric.objects.filter(
|
||||
model_name="llama3.2"
|
||||
).values_list("prompt_id", flat=True)
|
||||
)
|
||||
self.assertEqual(joined.count(), 1)
|
||||
|
||||
def test_unauthenticated_rejected(self):
|
||||
self.client.force_authenticate(user=None)
|
||||
response = self.client.post(
|
||||
self.url,
|
||||
{"prompt_id": self.assistant.id, "rating": "up"},
|
||||
format="json",
|
||||
)
|
||||
self.assertEqual(response.status_code, status.HTTP_401_UNAUTHORIZED)
|
||||
@@ -12,7 +12,6 @@ from .views import (
|
||||
is_authenticated,
|
||||
AnnouncmentView,
|
||||
FeedbackView,
|
||||
PromptFeedbackView,
|
||||
ConversationsView,
|
||||
ConversationDetailView,
|
||||
CompanyUsersView,
|
||||
@@ -79,11 +78,6 @@ urlpatterns = [
|
||||
path("announcment/get/", AnnouncmentView.as_view(), name="get_announcments"),
|
||||
path("conversations", ConversationsView.as_view(), name="conversations"),
|
||||
path("feedbacks/", FeedbackView.as_view(), name="feedbacks"),
|
||||
path(
|
||||
"prompt_feedback",
|
||||
PromptFeedbackView.as_view(),
|
||||
name="prompt_feedback",
|
||||
),
|
||||
path(
|
||||
"conversation_details",
|
||||
ConversationDetailView.as_view(),
|
||||
|
||||
@@ -13,8 +13,6 @@ from .serializers import (
|
||||
ConversationSerializer,
|
||||
PromptSerializer,
|
||||
FeedbackSerializer,
|
||||
PromptFeedbackSerializer,
|
||||
PromptFeedbackUpsertSerializer,
|
||||
DocumentWorkspaceSerializer,
|
||||
DocumentSerializer,
|
||||
)
|
||||
@@ -26,7 +24,6 @@ from .models import (
|
||||
Announcement,
|
||||
Conversation,
|
||||
Prompt,
|
||||
PromptFeedback,
|
||||
Feedback,
|
||||
PromptMetric,
|
||||
DocumentWorkspace,
|
||||
@@ -383,90 +380,6 @@ class FeedbackView(APIView):
|
||||
return Response(serializer.data, status=status.HTTP_200_OK)
|
||||
|
||||
|
||||
def _user_can_rate_prompt(user, prompt: Prompt) -> bool:
|
||||
"""Caller may rate prompts in their own non-deleted conversations."""
|
||||
conversation = prompt.conversation
|
||||
return (
|
||||
conversation is not None
|
||||
and conversation.user_id == user.id
|
||||
and not conversation.deleted
|
||||
)
|
||||
|
||||
|
||||
class PromptFeedbackView(APIView):
|
||||
"""Upsert / clear per-message thumbs ratings (chat_backend#67)."""
|
||||
|
||||
http_method_names = ["post", "delete"]
|
||||
|
||||
def post(self, request, format="json"):
|
||||
serializer = PromptFeedbackUpsertSerializer(data=request.data)
|
||||
if not serializer.is_valid():
|
||||
return Response(serializer.errors, status=status.HTTP_400_BAD_REQUEST)
|
||||
|
||||
prompt_id = serializer.validated_data["prompt_id"]
|
||||
try:
|
||||
prompt = Prompt.objects.select_related("conversation").get(id=prompt_id)
|
||||
except Prompt.DoesNotExist:
|
||||
return Response(
|
||||
{"detail": "Prompt not found."},
|
||||
status=status.HTTP_404_NOT_FOUND,
|
||||
)
|
||||
|
||||
if not _user_can_rate_prompt(request.user, prompt):
|
||||
return Response(
|
||||
{"detail": "Prompt not found."},
|
||||
status=status.HTTP_404_NOT_FOUND,
|
||||
)
|
||||
|
||||
if prompt.user_created:
|
||||
return Response(
|
||||
{"detail": "Only assistant prompts can be rated."},
|
||||
status=status.HTTP_400_BAD_REQUEST,
|
||||
)
|
||||
|
||||
feedback, _created = PromptFeedback.objects.update_or_create(
|
||||
prompt=prompt,
|
||||
user=request.user,
|
||||
defaults={
|
||||
"rating": serializer.validated_data["rating"],
|
||||
"reason": serializer.validated_data.get("reason"),
|
||||
"comment": serializer.validated_data.get("comment"),
|
||||
},
|
||||
)
|
||||
return Response(
|
||||
PromptFeedbackSerializer(feedback).data,
|
||||
status=status.HTTP_200_OK,
|
||||
)
|
||||
|
||||
def delete(self, request, format="json"):
|
||||
prompt_id = request.query_params.get("prompt_id")
|
||||
if prompt_id is None:
|
||||
return Response(
|
||||
{"detail": "prompt_id is required."},
|
||||
status=status.HTTP_400_BAD_REQUEST,
|
||||
)
|
||||
try:
|
||||
prompt_id = int(prompt_id)
|
||||
except (TypeError, ValueError):
|
||||
return Response(
|
||||
{"detail": "prompt_id must be an integer."},
|
||||
status=status.HTTP_400_BAD_REQUEST,
|
||||
)
|
||||
|
||||
deleted, _ = PromptFeedback.objects.filter(
|
||||
prompt_id=prompt_id,
|
||||
user=request.user,
|
||||
prompt__conversation__user=request.user,
|
||||
prompt__conversation__deleted=False,
|
||||
).delete()
|
||||
if not deleted:
|
||||
return Response(
|
||||
{"detail": "Prompt feedback not found."},
|
||||
status=status.HTTP_404_NOT_FOUND,
|
||||
)
|
||||
return Response(status=status.HTTP_204_NO_CONTENT)
|
||||
|
||||
|
||||
class AcknowledgeTermsOfService(APIView):
|
||||
http_method_names = ["post"]
|
||||
|
||||
@@ -603,20 +516,11 @@ class ConversationDetailView(APIView):
|
||||
{"detail": "Conversation not found."},
|
||||
status=status.HTTP_404_NOT_FOUND,
|
||||
)
|
||||
prompts = list(
|
||||
Prompt.objects.filter(
|
||||
prompts = Prompt.objects.filter(
|
||||
conversation__id=conversation_id, conversation__user=request.user
|
||||
)
|
||||
)
|
||||
serializer = PromptSerializer(
|
||||
prompts,
|
||||
many=True,
|
||||
context={
|
||||
"request": request,
|
||||
"_prompt_ids_for_feedback": [p.id for p in prompts],
|
||||
},
|
||||
)
|
||||
return Response(serializer.data, status=status.HTTP_200_OK)
|
||||
serailzer = PromptSerializer(prompts, many=True)
|
||||
return Response(serailzer.data, status=status.HTTP_200_OK)
|
||||
|
||||
def post(self, request, format="json"):
|
||||
logger.info("In the post")
|
||||
|
||||
Reference in New Issue
Block a user