From 87fd81637e6d41b3f2480c01b236a9912a06416f Mon Sep 17 00:00:00 2001 From: Dan LaManna Date: Mon, 30 Sep 2024 12:19:49 -0400 Subject: [PATCH] Remove incorrect log name constraint --- .../0003_alter_lastenqueueds3log_name.py | 17 +++++++++++++++++ isic/stats/models.py | 12 +++--------- isic/stats/tests/test_tasks.py | 8 +++++--- 3 files changed, 25 insertions(+), 12 deletions(-) create mode 100644 isic/stats/migrations/0003_alter_lastenqueueds3log_name.py diff --git a/isic/stats/migrations/0003_alter_lastenqueueds3log_name.py b/isic/stats/migrations/0003_alter_lastenqueueds3log_name.py new file mode 100644 index 00000000..f42513ef --- /dev/null +++ b/isic/stats/migrations/0003_alter_lastenqueueds3log_name.py @@ -0,0 +1,17 @@ +# Generated by Django 5.1.1 on 2024-09-30 16:19 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + dependencies = [ + ("stats", "0002_lastenqueueds3log"), + ] + + operations = [ + migrations.AlterField( + model_name="lastenqueueds3log", + name="name", + field=models.CharField(unique=True), + ), + ] diff --git a/isic/stats/models.py b/isic/stats/models.py index 3927fb28..0cb2b626 100644 --- a/isic/stats/models.py +++ b/isic/stats/models.py @@ -1,4 +1,3 @@ -from django.core.validators import RegexValidator from django.db import models from django.db.models.constraints import CheckConstraint, UniqueConstraint from django.db.models.expressions import F @@ -52,14 +51,9 @@ class LastEnqueuedS3Log(models.Model): This table is intended to only have one row. """ - name = models.CharField( - max_length=36, - validators=[ - # https://docs.aws.amazon.com/AmazonS3/latest/userguide/ServerLogs.html#server-log-keyname-format - RegexValidator(r"^\d{4}-(\d{2}-){5}[A-F0-9]{16}$") - ], - unique=True, - ) + # Stores keys in the format referenced here: + # https://docs.aws.amazon.com/AmazonCloudFront/latest/DeveloperGuide/AccessLogs.html#AccessLogsFileNaming + name = models.CharField(unique=True) def __str__(self) -> str: return self.name diff --git a/isic/stats/tests/test_tasks.py b/isic/stats/tests/test_tasks.py index 2f8bcdbe..b4c1e7b9 100644 --- a/isic/stats/tests/test_tasks.py +++ b/isic/stats/tests/test_tasks.py @@ -74,6 +74,8 @@ def test_cdn_access_log_parsing(mocker): def test_collect_image_download_records_task( mocker, image_factory, django_capture_on_commit_callbacks ): + KEY_NAME = "E1GEV1F1LW8QVF.2024-09-16-11.c93bad02.gz" # noqa: N806 + # TODO: overriding the blob name requires passing the size manually. image = image_factory( accession__blob="some/exists.jpg", @@ -89,7 +91,7 @@ def _delete_object(*args, **kwargs): return mocker.MagicMock(delete_object=_delete_object) mocker.patch("isic.stats.tasks.boto3", mocker.MagicMock(client=mock_client)) - mocker.patch("isic.stats.tasks._cdn_log_objects", return_value=[{"Key": "foo"}]) + mocker.patch("isic.stats.tasks._cdn_log_objects", return_value=[{"Key": KEY_NAME}]) mocker.patch("isic.stats.tasks.BytesIO", mocker.MagicMock()) mocker.patch( "isic.stats.tasks._cdn_access_log_records", @@ -143,7 +145,7 @@ def _delete_object(*args, **kwargs): assert ImageDownload.objects.count() == 1 assert image.downloads.count() == 1 - # assert that re-running the task only looks for new logs after "foo" + # assert that re-running the task only looks for new logs after the first key import isic.stats.tasks cdn_log_objects = mocker.spy(isic.stats.tasks, "_cdn_log_objects") @@ -155,4 +157,4 @@ def _delete_object(*args, **kwargs): assert image.downloads.count() == 1 assert cdn_log_objects.call_count == 1 - assert cdn_log_objects.call_args[0][1] == "foo" + assert cdn_log_objects.call_args[0][1] == KEY_NAME