From b1e952044e40985faa5f484d7906118f628d7436 Mon Sep 17 00:00:00 2001 From: Dan LaManna Date: Fri, 6 Sep 2024 13:27:12 -0400 Subject: [PATCH] More reliably disable cachalot for elasticsearch syncing --- isic/core/search.py | 14 +++++++++----- isic/core/tasks.py | 4 +--- 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/isic/core/search.py b/isic/core/search.py index c69ec0e7..a6e9fe36 100644 --- a/isic/core/search.py +++ b/isic/core/search.py @@ -3,6 +3,7 @@ import logging from typing import TYPE_CHECKING, Any +from cachalot.api import cachalot_disabled from django.conf import settings from django.contrib.auth.models import User from django.db.models.query import QuerySet @@ -108,13 +109,16 @@ def add_to_search_index(image: Image) -> None: def bulk_add_to_search_index(qs: QuerySet[Image], chunk_size: int = 2_000) -> None: from opensearchpy.helpers import parallel_bulk - # qs must be generated with with_elasticsearch_properties - # Use a generator for lazy evaluation - image_documents = (image.to_elasticsearch_document() for image in qs) - # The opensearch logger is very noisy when updating records, # set it to warning during this operation. - with LoggingContext(logging.getLogger("opensearch"), level=logging.WARNING): + with ( + LoggingContext(logging.getLogger("opensearch"), level=logging.WARNING), + cachalot_disabled(), + ): + # qs must be generated with with_elasticsearch_properties + # Use a generator for lazy evaluation + image_documents = (image.to_elasticsearch_document() for image in qs.iterator()) + for success, info in parallel_bulk( client=get_elasticsearch_client(), index=settings.ISIC_ELASTICSEARCH_INDEX, diff --git a/isic/core/tasks.py b/isic/core/tasks.py index d9bdbe67..e51b2384 100644 --- a/isic/core/tasks.py +++ b/isic/core/tasks.py @@ -5,7 +5,6 @@ from typing import cast import uuid -from cachalot.api import cachalot_disabled from celery import shared_task from django.conf import settings from django.contrib.auth.models import User @@ -60,8 +59,7 @@ def share_collection_with_users_task(collection_pk: int, grantor_pk: int, user_p retry_kwargs={"max_retries": 15}, ) def sync_elasticsearch_index_task(): - with cachalot_disabled(): - bulk_add_to_search_index(Image.objects.with_elasticsearch_properties().iterator()) + bulk_add_to_search_index(Image.objects.with_elasticsearch_properties()) @shared_task(soft_time_limit=1800, time_limit=1810)