Add new changes and Apis for extracts

Change latest prompt with used excerpts
Add OpenAiChat class for AzureOpenAI
Handle Empty df for Ops and recursive function for summary

main/permissions.py

@@ -20,6 +20,10 @@ class DenyGuestUserMutationPermission(permissions.BasePermission):
     """
 
     def _has_permission(self, request, view):
+        # Allow all safe methods (GET, HEAD, OPTIONS) which are non-mutating.
+        if request.method in permissions.SAFE_METHODS:
+            return True
+
         # For mutation methods (POST, PUT, DELETE, etc.):
         # Check if the user is authenticated.
         if not bool(request.user and request.user.is_authenticated):

main/urls.py

@@ -117,7 +117,6 @@
 router.register(r"public-per-stats", per_views.CountryPublicPerStatsViewset, basename="public_country_per_stats")
 router.register(r"per-stats", per_views.CountryPerStatsViewset, basename="country_per_stats")
 router.register(r"ops-learning", per_views.OpsLearningViewset, basename="ops_learning")
-router.register(r"ops-learning-summary", per_views.OpsLearningSummaryViewset, basename="ops_learning_summary")
 router.register(r"per-document-upload", per_views.PerDocumentUploadViewSet, basename="per_document_upload")
 
 router.register(r"personnel_deployment", deployment_views.PersonnelDeploymentViewset, basename="personnel_deployment")

per/cache.py

@@ -4,8 +4,13 @@
 
 import django_filters
 from django.core.serializers.json import DjangoJSONEncoder
+from django.db.models import Count, Prefetch
 
-from per.models import OpsLearningCacheResponse
+from per.models import (
+    OpsLearningCacheResponse,
+    OpsLearningComponentCacheResponse,
+    OpsLearningSectorCacheResponse,
+)
 
 
 class OpslearningSummaryCacheHelper:
@@ -47,15 +52,24 @@ def get_or_create(
         }
         hash_value = self.generate_hash(filter_data)
         # Check if the summary is already cached
-        ops_learning_summary, created = OpsLearningCacheResponse.objects.get_or_create(
+        # NOTE: count for the related components and sectors are prefetched
+        ops_learning_summary, created = OpsLearningCacheResponse.objects.prefetch_related(
+            "used_ops_learning",
+            Prefetch(
+                "ops_learning_component",
+                queryset=OpsLearningComponentCacheResponse.objects.select_related(
+                    "component",
+                ).annotate(count=Count("used_ops_learning")),
+            ),
+            Prefetch(
+                "ops_learning_sector",
+                queryset=OpsLearningSectorCacheResponse.objects.select_related(
+                    "sector",
+                ).annotate(count=Count("used_ops_learning")),
+            ),
+        ).get_or_create(
             used_filters_hash=hash_value,
             used_filters=filter_data,
-            status=OpsLearningCacheResponse.Status.SUCCESS,
             defaults={"status": OpsLearningCacheResponse.Status.PENDING},
         )
-        if not created:
-            return ops_learning_summary
-        # TODO send a http code of task is pending and return the task id
-        # transaction.on_commit(lambda: generate_summary.delay(ops_learning_summary, filter_data))
-        # return Response({"task_id": ops_learning_summary.id}, status=202)
-        return OpsLearningCacheResponse.objects.filter(status=OpsLearningCacheResponse.Status.SUCCESS).first()
+        return ops_learning_summary, filter_data

per/drf_views.py

@@ -2,6 +2,7 @@
 
 import pytz
 from django.conf import settings
+from django.db import transaction
 from django.db.models import Prefetch, Q
 from django.http import HttpResponse
 from django.shortcuts import get_object_or_404
@@ -34,6 +35,7 @@
     PerGeneralPermission,
     PerPermission,
 )
+from per.task import generate_summary
 from per.utils import filter_per_queryset_by_user_access
 
 from .admin_classes import RegionRestrictedAdmin
@@ -54,6 +56,8 @@
     NiceDocument,
     OpsLearning,
     OpsLearningCacheResponse,
+    OpsLearningComponentCacheResponse,
+    OpsLearningSectorCacheResponse,
     OrganizationTypes,
     Overview,
     PerAssessment,
@@ -678,6 +682,18 @@ class OpsLearningFilter(filters.FilterSet):
         widget=CSVWidget,
         queryset=FormComponent.objects.all(),
     )
+    insight_id = filters.NumberFilter(
+        label="Base Insight id for used extracts",
+        method="get_cache_response",
+    )
+    insight_sector_id = filters.NumberFilter(label="Sector insight id for used extracts", method="get_cache_response_sector")
+    insight_component_id = filters.NumberFilter(
+        label="Component insight id for used extracts",
+        method="get_cache_response_component",
+    )
+    # NOTE: overriding the fields for the typing issue
+    sector_validated = filters.NumberFilter(field_name="sector_validated", lookup_expr="exact")
+    per_component_validated = filters.NumberFilter(field_name="per_component_validated", lookup_expr="exact")
 
     class Meta:
         model = OpsLearning
@@ -689,8 +705,6 @@ class Meta:
             "learning": ("exact", "icontains"),
             "learning_validated": ("exact", "icontains"),
             "organization_validated": ("exact",),
-            "sector_validated": ("exact",),
-            "per_component_validated": ("exact",),
             "appeal_code": ("exact", "in"),
             "appeal_code__code": ("exact", "icontains", "in"),
             "appeal_code__num_beneficiaries": ("exact", "gt", "gte", "lt", "lte"),
@@ -704,6 +718,23 @@ class Meta:
             "appeal_code__region": ("exact", "in"),
         }
 
+    def get_cache_response(self, queryset, name, value):
+        if value and (ops_learning_cache_response := OpsLearningCacheResponse.objects.filter(id=value).first()):
+            return queryset.filter(id__in=ops_learning_cache_response.used_ops_learning.all())
+        return queryset
+
+    def get_cache_response_sector(self, queryset, name, value):
+        if value and (ops_learning_sector_cache_response := OpsLearningSectorCacheResponse.objects.filter(id=value).first()):
+            return queryset.filter(id__in=ops_learning_sector_cache_response.used_ops_learning.all())
+        return queryset
+
+    def get_cache_response_component(self, queryset, name, value):
+        if value and (
+            ops_learning_component_cache_response := OpsLearningComponentCacheResponse.objects.filter(id=value).first()
+        ):
+            return queryset.filter(id__in=ops_learning_component_cache_response.used_ops_learning.all())
+        return queryset
+
 
 class OpsLearningViewset(viewsets.ModelViewSet):
     """
@@ -736,15 +767,27 @@ def get_queryset(self):
             return qs.select_related(
                 "appeal_code",
             ).prefetch_related(
-                "sector", "organization", "per_component", "sector_validated", "organization_validated", "per_component_validated"
+                "sector",
+                "organization",
+                "per_component",
+                "sector_validated",
+                "organization_validated",
+                "per_component_validated",
+                "appeal_code__event__countries_for_preview",
             )
         return (
             qs.filter(is_validated=True)
             .select_related(
                 "appeal_code",
             )
             .prefetch_related(
-                "sector", "organization", "per_component", "sector_validated", "organization_validated", "per_component_validated"
+                "sector",
+                "organization",
+                "per_component",
+                "sector_validated",
+                "organization_validated",
+                "per_component_validated",
+                "appeal_code__event__countries_for_preview",
             )
         )
 
@@ -813,7 +856,7 @@ def get_renderer_context(self):
     @extend_schema(
         request=None,
         filters=True,
-        responses=OpsLearningSummarySerializer(),
+        responses=OpsLearningSummarySerializer,
     )
     @action(
         detail=False,
@@ -825,7 +868,11 @@ def summary(self, request):
         """
         Get the Ops Learning Summary based on the filters
         """
-        ops_learning_summary_instance = OpslearningSummaryCacheHelper.get_or_create(request, [self.filterset_class])
+        ops_learning_summary_instance, filter_data = OpslearningSummaryCacheHelper.get_or_create(request, [self.filterset_class])
+        if ops_learning_summary_instance.status == OpsLearningCacheResponse.Status.SUCCESS:
+            return response.Response(OpsLearningSummarySerializer(ops_learning_summary_instance).data)
+
+        transaction.on_commit(lambda: generate_summary.delay(ops_learning_summary_instance.id, filter_data))
         return response.Response(OpsLearningSummarySerializer(ops_learning_summary_instance).data)
 
 
@@ -839,10 +886,3 @@ def get_queryset(self):
         queryset = super().get_queryset()
         user = self.request.user
         return filter_per_queryset_by_user_access(user, queryset)
-
-
-class OpsLearningSummaryViewset(viewsets.ReadOnlyModelViewSet):
-    queryset = OpsLearningCacheResponse.objects.all()
-    serializer_class = OpsLearningSummarySerializer
-    permission_classes = [permissions.IsAuthenticated]
-    pagination_class = None

per/factories.py

@@ -64,7 +64,7 @@ class Meta:
 
 class FormComponentFactory(factory.django.DjangoModelFactory):
     area = factory.SubFactory(FormAreaFactory)
-    title = fuzzy.FuzzyText(length=50, prefix="component-")
+    title = factory.Faker("sentence", nb_words=5)
 
     class Meta:
         model = FormComponent
@@ -114,20 +114,20 @@ class Meta:
 
 class OpsLearningCacheResponseFactory(factory.django.DjangoModelFactory):
     used_filters_hash = fuzzy.FuzzyText(length=20)
-    insights1_title = fuzzy.FuzzyText(length=50, prefix="insights1-title-")
-    insights1_content = fuzzy.FuzzyText(length=100, prefix="insights1-content-")
-    insights2_title = fuzzy.FuzzyText(length=50, prefix="insights2-title-")
-    insights2_content = fuzzy.FuzzyText(length=100, prefix="insights2-content-")
-    insights3_title = fuzzy.FuzzyText(length=50, prefix="insights3-title-")
-    insights3_content = fuzzy.FuzzyText(length=100, prefix="insights3-content-")
+    insights1_title = factory.Faker("sentence", nb_words=5)
+    insights1_content = factory.Faker("sentence", nb_words=20)
+    insights2_title = factory.Faker("sentence", nb_words=5)
+    insights2_content = factory.Faker("sentence", nb_words=25)
+    insights3_title = factory.Faker("sentence", nb_words=10)
+    insights3_content = factory.Faker("sentence", nb_words=30)
 
     class Meta:
         model = OpsLearningCacheResponse
 
 
 class OpsLearningSectorCacheResponseFactory(factory.django.DjangoModelFactory):
     filter_response = factory.SubFactory(OpsLearningCacheResponseFactory)
-    content = fuzzy.FuzzyText(length=50)
+    content = factory.Faker("sentence", nb_words=30)
     sector = factory.SubFactory(SectorTagFactory)
 
     class Meta:
@@ -136,7 +136,7 @@ class Meta:
 
 class OpsLearningComponentCacheResponseFactory(factory.django.DjangoModelFactory):
     filter_response = factory.SubFactory(OpsLearningCacheResponseFactory)
-    content = fuzzy.FuzzyText(length=50)
+    content = factory.Faker("sentence", nb_words=30)
     component = factory.SubFactory(FormComponentFactory)
 
     class Meta:

per/migrations/0121_opslearningcacheresponse_and_more.py

@@ -1,4 +1,4 @@
-# Generated by Django 4.2.14 on 2024-08-02 10:51
+# Generated by Django 4.2.15 on 2024-08-23 03:54
 
 import django.db.models.deletion
 from django.db import migrations, models
@@ -21,7 +21,9 @@ class Migration(migrations.Migration):
                 (
                     "status",
                     models.IntegerField(
-                        choices=[(0, "pending"), (1, "started"), (2, "success"), (3, "failed")], default=0, verbose_name="status"
+                        choices=[(1, "Pending"), (2, "Started"), (3, "Success"), (4, "No extract available"), (5, "Failed")],
+                        default=1,
+                        verbose_name="status",
                     ),
                 ),
                 ("insights1_content", models.TextField(blank=True, null=True, verbose_name="insights 1")),
@@ -32,15 +34,15 @@ class Migration(migrations.Migration):
                 ("insights3_title", models.CharField(blank=True, max_length=255, null=True, verbose_name="insights 3 title")),
                 (
                     "insights1_confidence_level",
-                    models.CharField(blank=True, null=True, verbose_name="insights 1 confidence level"),
+                    models.CharField(blank=True, max_length=10, null=True, verbose_name="insights 1 confidence level"),
                 ),
                 (
                     "insights2_confidence_level",
-                    models.CharField(blank=True, null=True, verbose_name="insights 2 confidence level"),
+                    models.CharField(blank=True, max_length=10, null=True, verbose_name="insights 2 confidence level"),
                 ),
                 (
                     "insights3_confidence_level",
-                    models.CharField(blank=True, null=True, verbose_name="insights 3 confidence level"),
+                    models.CharField(blank=True, max_length=10, null=True, verbose_name="insights 3 confidence level"),
                 ),
                 ("contradictory_reports", models.TextField(blank=True, null=True, verbose_name="contradictory reports")),
                 ("modified_at", models.DateTimeField(auto_now=True, verbose_name="modified_at")),
@@ -54,15 +56,15 @@ class Migration(migrations.Migration):
                 ("id", models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name="ID")),
                 ("prompt_hash", models.CharField(max_length=32, verbose_name="used prompt hash")),
                 ("prompt", models.TextField(blank=True, null=True, verbose_name="used prompt")),
-                ("type", models.IntegerField(choices=[(0, "primary"), (1, "secondary")], verbose_name="type")),
+                ("type", models.IntegerField(choices=[(1, "Primary"), (2, "Secondary")], verbose_name="type")),
                 ("response", models.JSONField(default=dict, verbose_name="response")),
             ],
         ),
         migrations.CreateModel(
             name="OpsLearningSectorCacheResponse",
             fields=[
                 ("id", models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name="ID")),
-                ("content", models.TextField(verbose_name="content")),
+                ("content", models.TextField(blank=True, null=True, verbose_name="content")),
                 (
                     "filter_response",
                     models.ForeignKey(
@@ -88,7 +90,7 @@ class Migration(migrations.Migration):
             name="OpsLearningComponentCacheResponse",
             fields=[
                 ("id", models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name="ID")),
-                ("content", models.TextField(verbose_name="content")),
+                ("content", models.TextField(blank=True, null=True, verbose_name="content")),
                 (
                     "component",
                     models.ForeignKey(

per/models.py

@@ -753,8 +753,8 @@ def __str__(self):
 
 class OpsLearningPromptResponseCache(models.Model):
     class PromptType(models.IntegerChoices):
-        PRIMARY = 0, _("primary")
-        SECONDARY = 1, _("secondary")
+        PRIMARY = 1, _("Primary")
+        SECONDARY = 2, _("Secondary")
 
     prompt_hash = models.CharField(verbose_name=_("used prompt hash"), max_length=32)
     prompt = models.TextField(verbose_name=_("used prompt"), null=True, blank=True)
@@ -768,10 +768,11 @@ def __str__(self) -> str:
 
 class OpsLearningCacheResponse(models.Model):
     class Status(models.IntegerChoices):
-        PENDING = 0, _("pending")
-        STARTED = 1, _("started")
-        SUCCESS = 2, _("success")
-        FAILED = 3, _("failed")
+        PENDING = 1, _("Pending")
+        STARTED = 2, _("Started")
+        SUCCESS = 3, _("Success")
+        NO_EXTRACT_AVAILABLE = 4, _("No extract available")
+        FAILED = 5, _("Failed")
 
     used_filters_hash = models.CharField(verbose_name=_("used filters hash"), max_length=32)
     used_filters = models.JSONField(verbose_name=_("used filters"), default=dict)
@@ -786,10 +787,15 @@ class Status(models.IntegerChoices):
     insights2_title = models.CharField(verbose_name=_("insights 2 title"), max_length=255, null=True, blank=True)
     insights3_title = models.CharField(verbose_name=_("insights 3 title"), max_length=255, null=True, blank=True)
 
-    insights1_confidence_level = models.CharField(verbose_name=_("insights 1 confidence level"), null=True, blank=True)
-    insights2_confidence_level = models.CharField(verbose_name=_("insights 2 confidence level"), null=True, blank=True)
-    insights3_confidence_level = models.CharField(verbose_name=_("insights 3 confidence level"), null=True, blank=True)
-
+    insights1_confidence_level = models.CharField(
+        verbose_name=_("insights 1 confidence level"), max_length=10, null=True, blank=True
+    )
+    insights2_confidence_level = models.CharField(
+        verbose_name=_("insights 2 confidence level"), max_length=10, null=True, blank=True
+    )
+    insights3_confidence_level = models.CharField(
+        verbose_name=_("insights 3 confidence level"), max_length=10, null=True, blank=True
+    )
     contradictory_reports = models.TextField(verbose_name=_("contradictory reports"), null=True, blank=True)
 
     used_ops_learning = models.ManyToManyField(
@@ -816,14 +822,14 @@ class OpsLearningSectorCacheResponse(models.Model):
         on_delete=models.CASCADE,
         related_name="+",
     )
-    content = models.TextField(verbose_name=_("content"))
+    content = models.TextField(verbose_name=_("content"), null=True, blank=True)
     used_ops_learning = models.ManyToManyField(
         OpsLearning,
         related_name="+",
     )
 
     def __str__(self) -> str:
-        return f"sector - {self.content}"
+        return f"Summary - sector - {self.sector.title}"
 
 
 class OpsLearningComponentCacheResponse(models.Model):
@@ -839,11 +845,11 @@ class OpsLearningComponentCacheResponse(models.Model):
         on_delete=models.CASCADE,
         related_name="+",
     )
-    content = models.TextField(verbose_name=_("content"))
+    content = models.TextField(verbose_name=_("content"), null=True, blank=True)
     used_ops_learning = models.ManyToManyField(
         OpsLearning,
         related_name="+",
     )
 
     def __str__(self) -> str:
-        return f"component - {self.content}"
+        return f"Summary - component - {self.component.title}"