From 63dee3c55a9ddc3f208cfb8c388cdc1b06951701 Mon Sep 17 00:00:00 2001 From: Nathan Voxland Date: Tue, 15 Aug 2023 14:55:53 -0500 Subject: [PATCH] Disable locks progressbar in tests --- deeplake/api/dataset.py | 14 ++++++++------ deeplake/api/tests/test_api.py | 4 ++-- deeplake/auto/structured/dataframe.py | 6 ++++-- deeplake/auto/unstructured/base.py | 7 ++++++- deeplake/auto/unstructured/coco/coco.py | 3 ++- .../auto/unstructured/image_classification.py | 4 +++- deeplake/auto/unstructured/yolo/yolo.py | 11 +++++++++-- deeplake/constants.py | 1 + deeplake/core/dataset/dataset.py | 8 +++++--- deeplake/core/query/filter.py | 8 +++++--- deeplake/core/query/test/test_query.py | 4 ++-- deeplake/core/transform/transform.py | 9 ++++++--- .../vectorstore/test_deeplake_vectorstore.py | 2 -- .../integrations/huggingface/huggingface.py | 5 ++++- deeplake/tests/dataset_fixtures.py | 18 +++++++++++++++++- deeplake/util/class_label.py | 2 +- 16 files changed, 75 insertions(+), 31 deletions(-) diff --git a/deeplake/api/dataset.py b/deeplake/api/dataset.py index 1e9be1e2f4..c632bd1920 100644 --- a/deeplake/api/dataset.py +++ b/deeplake/api/dataset.py @@ -1,5 +1,7 @@ import os +from deeplake import constants + import deeplake import pathlib import posixpath @@ -1101,7 +1103,7 @@ def deepcopy( token=None, num_workers: int = 0, scheduler="threaded", - progressbar=True, + progressbar=constants.PROGRESSBAR_ENABLED_DEFAULT, public: bool = False, verbose: bool = True, **kwargs, @@ -1402,7 +1404,7 @@ def ingest_coco( src_creds: Optional[Union[str, Dict]] = None, dest_creds: Optional[Union[str, Dict]] = None, inspect_limit: int = 1000000, - progressbar: bool = True, + progressbar: bool = constants.PROGRESSBAR_ENABLED_DEFAULT, shuffle: bool = False, num_workers: int = 0, token: Optional[str] = None, @@ -1524,7 +1526,7 @@ def ingest_yolo( dest_creds: Optional[Union[str, Dict]] = None, image_creds_key: Optional[str] = None, inspect_limit: int = 1000, - progressbar: bool = True, + progressbar: bool = constants.PROGRESSBAR_ENABLED_DEFAULT, shuffle: bool = False, num_workers: int = 0, token: Optional[str] = None, @@ -1648,7 +1650,7 @@ def ingest_classification( image_params: Optional[Dict] = None, label_params: Optional[Dict] = None, dest_creds: Optional[Union[str, Dict]] = None, - progressbar: bool = True, + progressbar: bool = constants.PROGRESSBAR_ENABLED_DEFAULT, summary: bool = True, num_workers: int = 0, shuffle: bool = True, @@ -1805,7 +1807,7 @@ def ingest_kaggle( images_compression: str = "auto", dest_creds: Optional[Union[str, Dict]] = None, kaggle_credentials: Optional[dict] = None, - progressbar: bool = True, + progressbar: bool = constants.PROGRESSBAR_ENABLED_DEFAULT, summary: bool = True, shuffle: bool = True, **dataset_kwargs, @@ -1885,7 +1887,7 @@ def ingest_dataframe( src_creds: Optional[Union[str, Dict]] = None, dest_creds: Optional[Union[str, Dict]] = None, creds_key: Optional[Dict] = None, - progressbar: bool = True, + progressbar: bool = constants.PROGRESSBAR_ENABLED_DEFAULT, token: Optional[str] = None, connect_kwargs: Optional[Dict] = None, **dataset_kwargs, diff --git a/deeplake/api/tests/test_api.py b/deeplake/api/tests/test_api.py index 6351bf469c..2f1c54afa3 100644 --- a/deeplake/api/tests/test_api.py +++ b/deeplake/api/tests/test_api.py @@ -976,7 +976,7 @@ def test_dataset_rename(ds_generator, path, hub_token, convert_to_pathlib): indirect=True, ) @pytest.mark.parametrize("num_workers", [2]) -@pytest.mark.parametrize("progressbar", [True]) +@pytest.mark.parametrize("progressbar", [False]) def test_dataset_deepcopy(path, hub_token, num_workers, progressbar): src_path = "_".join((path, "src1")) dest_path = "_".join((path, "dest1")) @@ -1877,7 +1877,7 @@ def test_hidden_tensors(local_ds_generator): @pytest.mark.parametrize("num_workers", [0, 2]) -@pytest.mark.parametrize("progressbar", [True, False]) +@pytest.mark.parametrize("progressbar", [False]) @pytest.mark.parametrize( "index", [slice(None), slice(5, None, None), slice(None, 8, 2), 7] ) diff --git a/deeplake/auto/structured/dataframe.py b/deeplake/auto/structured/dataframe.py index ca8dc35376..058cf57be1 100644 --- a/deeplake/auto/structured/dataframe.py +++ b/deeplake/auto/structured/dataframe.py @@ -1,6 +1,6 @@ import numpy as np from .base import StructuredDataset -from deeplake import Dataset +from deeplake import Dataset, constants from deeplake import read, link from deeplake.htype import HTYPE_SUPPORTED_COMPRESSIONS from deeplake.util.exceptions import IngestionError @@ -146,7 +146,9 @@ def _get_extend_values(self, tensor_params: dict, key: str): # type: ignore return extend_values - def fill_dataset(self, ds: Dataset, progressbar: bool = True) -> Dataset: + def fill_dataset( + self, ds: Dataset, progressbar: bool = constants.PROGRESSBAR_ENABLED_DEFAULT + ) -> Dataset: """Fill dataset with data from the dataframe - one tensor per column Args: diff --git a/deeplake/auto/unstructured/base.py b/deeplake/auto/unstructured/base.py index 255305b90d..6c07eed1e7 100644 --- a/deeplake/auto/unstructured/base.py +++ b/deeplake/auto/unstructured/base.py @@ -1,4 +1,7 @@ from abc import ABC, abstractmethod + +from deeplake import constants + from deeplake.util.path import find_root from pathlib import Path @@ -14,5 +17,7 @@ def __init__(self, source: str): """ @abstractmethod - def structure(ds, progressbar: bool = True, **kwargs): + def structure( + ds, progressbar: bool = constants.PROGRESSBAR_ENABLED_DEFAULT, **kwargs + ): pass diff --git a/deeplake/auto/unstructured/coco/coco.py b/deeplake/auto/unstructured/coco/coco.py index 5a688e06b1..80128cb0d4 100644 --- a/deeplake/auto/unstructured/coco/coco.py +++ b/deeplake/auto/unstructured/coco/coco.py @@ -3,6 +3,7 @@ from pathlib import Path from typing import List, Union, Dict, Optional from itertools import chain +from deeplake import constants from deeplake.core.dataset import Dataset from deeplake.core.tensor import Tensor @@ -139,7 +140,7 @@ def prepare_structure(self, inspect_limit: int = 1000000) -> DatasetStructure: self._structure = structure return structure - def structure(self, ds: Dataset, progressbar: bool = True, num_workers: int = 0, shuffle: bool = True): # type: ignore + def structure(self, ds: Dataset, progressbar: bool = constants.PROGRESSBAR_ENABLED_DEFAULT, num_workers: int = 0, shuffle: bool = True): # type: ignore image_files = self.images.supported_images if shuffle: diff --git a/deeplake/auto/unstructured/image_classification.py b/deeplake/auto/unstructured/image_classification.py index b1fc0f4c24..37a29b61cf 100644 --- a/deeplake/auto/unstructured/image_classification.py +++ b/deeplake/auto/unstructured/image_classification.py @@ -5,6 +5,8 @@ import glob from typing import List, Tuple, Union +from deeplake import constants + from deeplake.util.auto import ingestion_summary from deeplake.util.exceptions import ( InvalidPathException, @@ -92,7 +94,7 @@ def get_class_names(self) -> List[str]: def structure( # type: ignore self, ds: Dataset, - progressbar: bool = True, + progressbar: bool = constants.PROGRESSBAR_ENABLED_DEFAULT, generate_summary: bool = True, shuffle: bool = True, image_tensor_args: dict = {}, diff --git a/deeplake/auto/unstructured/yolo/yolo.py b/deeplake/auto/unstructured/yolo/yolo.py index 7686537ac1..2737061b1e 100644 --- a/deeplake/auto/unstructured/yolo/yolo.py +++ b/deeplake/auto/unstructured/yolo/yolo.py @@ -1,3 +1,5 @@ +from deeplake import constants + import deeplake from pathlib import Path @@ -218,7 +220,12 @@ def _add_images_tensor(self, structure: DatasetStructure): ) ) - def _ingest_data(self, ds: Dataset, progressbar: bool = True, num_workers: int = 0): + def _ingest_data( + self, + ds: Dataset, + progressbar: bool = constants.PROGRESSBAR_ENABLED_DEFAULT, + num_workers: int = 0, + ): """Functions appends the the data to the dataset object using deeplake.compute""" if self.image_creds_key is not None: @@ -298,7 +305,7 @@ def append_data_polygon(data, sample_out, tensor_meta: Dict = tensor_meta): num_workers=num_workers, ) - def structure(self, ds: Dataset, progressbar: bool = True, num_workers: int = 0, shuffle: bool = True): # type: ignore + def structure(self, ds: Dataset, progressbar: bool = constants.PROGRESSBAR_ENABLED_DEFAULT, num_workers: int = 0, shuffle: bool = True): # type: ignore # Set class names in the dataset if self.data.class_names: ds[self.label_params["name"]].info["class_names"] = self.data.class_names diff --git a/deeplake/constants.py b/deeplake/constants.py index 3a3784e879..dcf1c15d35 100644 --- a/deeplake/constants.py +++ b/deeplake/constants.py @@ -192,6 +192,7 @@ PYTEST_ENABLED = os.environ.get("DEEPLAKE_PYTEST_ENABLED", "").lower().strip() == "true" SPINNER_ENABLED = not PYTEST_ENABLED +PROGRESSBAR_ENABLED_DEFAULT = not PYTEST_ENABLED LOCK_LOCAL_DATASETS = not PYTEST_ENABLED diff --git a/deeplake/core/dataset/dataset.py b/deeplake/core/dataset/dataset.py index 75e755a518..f28bbcdb72 100644 --- a/deeplake/core/dataset/dataset.py +++ b/deeplake/core/dataset/dataset.py @@ -10,6 +10,8 @@ import pathlib import numpy as np from time import time, sleep + +from deeplake import constants from tqdm import tqdm # type: ignore import deeplake @@ -2167,7 +2169,7 @@ def filter( function: Union[Callable, str], num_workers: int = 0, scheduler: str = "threaded", - progressbar: bool = True, + progressbar: bool = constants.PROGRESSBAR_ENABLED_DEFAULT, save_result: bool = False, result_path: Optional[str] = None, result_ds_args: Optional[dict] = None, @@ -2874,7 +2876,7 @@ def rechunk( tensors: Optional[Union[str, List[str]]] = None, num_workers: int = 0, scheduler: str = "threaded", - progressbar: bool = True, + progressbar: bool = constants.PROGRESSBAR_ENABLED_DEFAULT, ): """Rewrites the underlying chunks to make their sizes optimal. This is usually needed in cases where a lot of updates have been made to the data. @@ -3752,7 +3754,7 @@ def load_view( tensors: Optional[List[str]] = None, num_workers: int = 0, scheduler: str = "threaded", - progressbar: Optional[bool] = True, + progressbar: Optional[bool] = constants.PROGRESSBAR_ENABLED_DEFAULT, ): """Loads the view and returns the :class:`~deeplake.core.dataset.dataset.Dataset` by id. Equivalent to ds.get_view(id).load(). diff --git a/deeplake/core/query/filter.py b/deeplake/core/query/filter.py index 2fb4f6f500..a53715d125 100644 --- a/deeplake/core/query/filter.py +++ b/deeplake/core/query/filter.py @@ -1,6 +1,8 @@ from typing import Callable, List, Optional, Sequence, Dict from uuid import uuid4 +from deeplake import constants + import deeplake from deeplake.core.io import SampleStreaming @@ -64,7 +66,7 @@ def filter_dataset( filter_function: Callable[[deeplake.Dataset], bool], num_workers: int = 0, scheduler: str = "threaded", - progressbar: bool = True, + progressbar: bool = constants.PROGRESSBAR_ENABLED_DEFAULT, save_result: bool = False, result_path: Optional[str] = None, result_ds_args: Optional[dict] = None, @@ -155,7 +157,7 @@ def filter_with_compute( filter_function: Callable, num_workers: int, scheduler: str, - progressbar: bool = True, + progressbar: bool = constants.PROGRESSBAR_ENABLED_DEFAULT, query_text: Optional[str] = None, vds: Optional[deeplake.Dataset] = None, ) -> List[int]: @@ -346,7 +348,7 @@ def query_dataset( query: str, num_workers: int = 0, scheduler: str = "threaded", - progressbar: bool = True, + progressbar: bool = constants.PROGRESSBAR_ENABLED_DEFAULT, save_result: bool = False, result_path: Optional[str] = None, result_ds_args: Optional[Dict] = None, diff --git a/deeplake/core/query/test/test_query.py b/deeplake/core/query/test/test_query.py index 56384e46ef..a1153cec32 100644 --- a/deeplake/core/query/test/test_query.py +++ b/deeplake/core/query/test/test_query.py @@ -123,8 +123,8 @@ def test_query_scheduler(local_ds): f1 = "labels % 2 == 0" f2 = lambda s: s.labels.numpy() % 2 == 0 - view1 = ds.filter(f1, num_workers=2, progressbar=True) - view2 = ds.filter(f2, num_workers=2, progressbar=True) + view1 = ds.filter(f1, num_workers=2) + view2 = ds.filter(f2, num_workers=2) np.testing.assert_array_equal(view1.labels.numpy(), view2.labels.numpy()) diff --git a/deeplake/core/transform/transform.py b/deeplake/core/transform/transform.py index 9c0839298b..dee15d6406 100644 --- a/deeplake/core/transform/transform.py +++ b/deeplake/core/transform/transform.py @@ -1,4 +1,7 @@ from uuid import uuid4 + +from deeplake import constants + import deeplake from typing import Callable, List, Optional from itertools import repeat @@ -57,7 +60,7 @@ def eval( ds_out: Optional[deeplake.Dataset] = None, num_workers: int = 0, scheduler: str = "threaded", - progressbar: bool = True, + progressbar: bool = constants.PROGRESSBAR_ENABLED_DEFAULT, skip_ok: bool = False, check_lengths: bool = True, pad_data_in: bool = False, @@ -135,7 +138,7 @@ def eval( ds_out: Optional[deeplake.Dataset] = None, num_workers: int = 0, scheduler: str = "threaded", - progressbar: bool = True, + progressbar: bool = constants.PROGRESSBAR_ENABLED_DEFAULT, skip_ok: bool = False, check_lengths: bool = True, pad_data_in: bool = False, @@ -340,7 +343,7 @@ def run( compute: ComputeProvider, num_workers: int, scheduler: str, - progressbar: bool = True, + progressbar: bool = constants.PROGRESSBAR_ENABLED_DEFAULT, overwrite: bool = False, skip_ok: bool = False, read_only: bool = False, diff --git a/deeplake/core/vectorstore/test_deeplake_vectorstore.py b/deeplake/core/vectorstore/test_deeplake_vectorstore.py index 277f957d40..1d828e15a9 100644 --- a/deeplake/core/vectorstore/test_deeplake_vectorstore.py +++ b/deeplake/core/vectorstore/test_deeplake_vectorstore.py @@ -1089,7 +1089,6 @@ def test_ingestion(local_path, capsys): vector_store = DeepLakeVectorStore( path=local_path, overwrite=True, - verbose=True, ) with pytest.raises(Exception): @@ -1210,7 +1209,6 @@ def test_ingestion_images(local_path): path=local_path, tensor_params=tensor_params, overwrite=True, - verbose=True, ) ids = vector_store.add(image=images, embedding=embeddings, return_ids=True) diff --git a/deeplake/integrations/huggingface/huggingface.py b/deeplake/integrations/huggingface/huggingface.py index 2aab7bca48..beee4d5418 100644 --- a/deeplake/integrations/huggingface/huggingface.py +++ b/deeplake/integrations/huggingface/huggingface.py @@ -1,5 +1,8 @@ import pathlib from typing import Dict, Set + +from deeplake import constants + from deeplake.core.dataset import Dataset import posixpath import deeplake @@ -69,7 +72,7 @@ def _create_tensor_from_feature(key, feature, src, ds): def ingest_huggingface( src, dest, - use_progressbar=True, + use_progressbar=constants.PROGRESSBAR_ENABLED_DEFAULT, token: Optional[str] = None, connect_kwargs: Optional[Dict] = None, **dataset_kwargs, diff --git a/deeplake/tests/dataset_fixtures.py b/deeplake/tests/dataset_fixtures.py index fc996f3d86..0a32dd7f9c 100644 --- a/deeplake/tests/dataset_fixtures.py +++ b/deeplake/tests/dataset_fixtures.py @@ -61,7 +61,7 @@ @pytest.fixture def memory_ds(memory_path): - return deeplake.dataset(memory_path) + return deeplake.dataset(memory_path, verbose=False) @pytest.fixture @@ -79,6 +79,8 @@ def local_ds_generator(local_path): def generate_local_ds(**kwargs): if kwargs.get("lock_enabled") is None: kwargs["lock_enabled"] = False + if kwargs.get("verbose") is None: + kwargs["verbose"] = False return deeplake.dataset(local_path, **kwargs) @@ -90,6 +92,8 @@ def local_auth_ds_generator(local_path, hub_cloud_dev_token): def generate_local_auth_ds(**kwargs): if kwargs.get("lock_enabled") is None: kwargs["lock_enabled"] = False + if kwargs.get("verbose") is None: + kwargs["verbose"] = False return deeplake.dataset(local_path, token=hub_cloud_dev_token, **kwargs) @@ -106,6 +110,8 @@ def s3_ds_generator(s3_path): def generate_s3_ds(**kwargs): if kwargs.get("lock_enabled") is None: kwargs["lock_enabled"] = False + if kwargs.get("verbose") is None: + kwargs["verbose"] = False return deeplake.dataset(s3_path, **kwargs) @@ -122,6 +128,8 @@ def gdrive_ds_generator(gdrive_path, gdrive_creds): def generate_gdrive_ds(**kwargs): if kwargs.get("lock_enabled") is None: kwargs["lock_enabled"] = False + if kwargs.get("verbose") is None: + kwargs["verbose"] = False return deeplake.dataset(gdrive_path, creds=gdrive_creds, **kwargs) @@ -138,6 +146,8 @@ def gcs_ds_generator(gcs_path, gcs_creds): def generate_gcs_ds(**kwargs): if kwargs.get("lock_enabled") is None: kwargs["lock_enabled"] = False + if kwargs.get("verbose") is None: + kwargs["verbose"] = False return deeplake.dataset(gcs_path, creds=gcs_creds, **kwargs) @@ -154,6 +164,8 @@ def azure_ds_generator(azure_path): def generate_azure_ds(**kwargs): if kwargs.get("lock_enabled") is None: kwargs["lock_enabled"] = False + if kwargs.get("verbose") is None: + kwargs["verbose"] = False return deeplake.dataset(azure_path, **kwargs) @@ -170,6 +182,8 @@ def hub_cloud_ds_generator(hub_cloud_path, hub_cloud_dev_token): def generate_hub_cloud_ds(**kwargs): if kwargs.get("lock_enabled") is None: kwargs["lock_enabled"] = False + if kwargs.get("verbose") is None: + kwargs["verbose"] = False return deeplake.dataset(hub_cloud_path, token=hub_cloud_dev_token, **kwargs) @@ -181,6 +195,8 @@ def hub_cloud_gcs_ds_generator(gcs_path, gcs_creds, hub_cloud_dev_token): def generate_hub_cloud_gcs_ds(**kwargs): if kwargs.get("lock_enabled") is None: kwargs["lock_enabled"] = False + if kwargs.get("verbose") is None: + kwargs["verbose"] = False ds = deeplake.dataset(gcs_path, creds=gcs_creds, **kwargs) ds.connect( diff --git a/deeplake/util/class_label.py b/deeplake/util/class_label.py index a862f13a7c..6e05c162cb 100644 --- a/deeplake/util/class_label.py +++ b/deeplake/util/class_label.py @@ -124,7 +124,7 @@ def class_label_sync( class_label_sync(label_tensor=tensor, hash_idx_map=hash_idx_map).eval( ds[temp_tensor], ds, - progressbar=True, + progressbar=verbose, check_lengths=False, skip_ok=True, )