Merge pull request #971 from ImageMarkup/fix-s3-log-name-format

Remove incorrect log name constraint

isic/stats/migrations/0003_alter_lastenqueueds3log_name.py

@@ -0,0 +1,17 @@
+# Generated by Django 5.1.1 on 2024-09-30 16:19
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("stats", "0002_lastenqueueds3log"),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name="lastenqueueds3log",
+            name="name",
+            field=models.CharField(unique=True),
+        ),
+    ]

isic/stats/models.py

@@ -1,4 +1,3 @@
-from django.core.validators import RegexValidator
 from django.db import models
 from django.db.models.constraints import CheckConstraint, UniqueConstraint
 from django.db.models.expressions import F
@@ -52,14 +51,9 @@ class LastEnqueuedS3Log(models.Model):
     This table is intended to only have one row.
     """
 
-    name = models.CharField(
-        max_length=36,
-        validators=[
-            # https://docs.aws.amazon.com/AmazonS3/latest/userguide/ServerLogs.html#server-log-keyname-format
-            RegexValidator(r"^\d{4}-(\d{2}-){5}[A-F0-9]{16}$")
-        ],
-        unique=True,
-    )
+    # Stores keys in the format referenced here:
+    # https://docs.aws.amazon.com/AmazonCloudFront/latest/DeveloperGuide/AccessLogs.html#AccessLogsFileNaming
+    name = models.CharField(unique=True)
 
     def __str__(self) -> str:
         return self.name

isic/stats/tests/test_tasks.py

@@ -74,6 +74,8 @@ def test_cdn_access_log_parsing(mocker):
 def test_collect_image_download_records_task(
     mocker, image_factory, django_capture_on_commit_callbacks
 ):
+    KEY_NAME = "E1GEV1F1LW8QVF.2024-09-16-11.c93bad02.gz"  # noqa: N806
+
     # TODO: overriding the blob name requires passing the size manually.
     image = image_factory(
         accession__blob="some/exists.jpg",
@@ -89,7 +91,7 @@ def _delete_object(*args, **kwargs):
         return mocker.MagicMock(delete_object=_delete_object)
 
     mocker.patch("isic.stats.tasks.boto3", mocker.MagicMock(client=mock_client))
-    mocker.patch("isic.stats.tasks._cdn_log_objects", return_value=[{"Key": "foo"}])
+    mocker.patch("isic.stats.tasks._cdn_log_objects", return_value=[{"Key": KEY_NAME}])
     mocker.patch("isic.stats.tasks.BytesIO", mocker.MagicMock())
     mocker.patch(
         "isic.stats.tasks._cdn_access_log_records",
@@ -143,7 +145,7 @@ def _delete_object(*args, **kwargs):
     assert ImageDownload.objects.count() == 1
     assert image.downloads.count() == 1
 
-    # assert that re-running the task only looks for new logs after "foo"
+    # assert that re-running the task only looks for new logs after the first key
     import isic.stats.tasks
 
     cdn_log_objects = mocker.spy(isic.stats.tasks, "_cdn_log_objects")
@@ -155,4 +157,4 @@ def _delete_object(*args, **kwargs):
     assert image.downloads.count() == 1
 
     assert cdn_log_objects.call_count == 1
-    assert cdn_log_objects.call_args[0][1] == "foo"
+    assert cdn_log_objects.call_args[0][1] == KEY_NAME