From ea90564e5f5276a37df06f7d3dadf90faa210d7f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 4 Dec 2023 09:13:27 +0100 Subject: [PATCH 001/130] Bump pypa/gh-action-pypi-publish from 1.8.10 to 1.8.11 (#1586) Bumps [pypa/gh-action-pypi-publish](https://github.com/pypa/gh-action-pypi-publish) from 1.8.10 to 1.8.11. - [Release notes](https://github.com/pypa/gh-action-pypi-publish/releases) - [Commits](https://github.com/pypa/gh-action-pypi-publish/compare/v1.8.10...v1.8.11) --- updated-dependencies: - dependency-name: pypa/gh-action-pypi-publish dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/releases.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/releases.yml b/.github/workflows/releases.yml index c08bfc6677..3bd25bfbf7 100644 --- a/.github/workflows/releases.yml +++ b/.github/workflows/releases.yml @@ -64,7 +64,7 @@ jobs: with: name: releases path: dist - - uses: pypa/gh-action-pypi-publish@v1.8.10 + - uses: pypa/gh-action-pypi-publish@v1.8.11 with: user: __token__ password: ${{ secrets.pypi_password }} From 79e80b36b14c50c6d522f0fe0caaee0bbfbce1a1 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 4 Dec 2023 09:13:52 +0100 Subject: [PATCH 002/130] Bump conda-incubator/setup-miniconda from 2.3.0 to 3.0.1 (#1587) Bumps [conda-incubator/setup-miniconda](https://github.com/conda-incubator/setup-miniconda) from 2.3.0 to 3.0.1. - [Release notes](https://github.com/conda-incubator/setup-miniconda/releases) - [Changelog](https://github.com/conda-incubator/setup-miniconda/blob/main/CHANGELOG.md) - [Commits](https://github.com/conda-incubator/setup-miniconda/compare/v2.3.0...v3.0.1) --- updated-dependencies: - dependency-name: conda-incubator/setup-miniconda dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/minimal.yml | 2 +- .github/workflows/python-package.yml | 2 +- .github/workflows/windows-testing.yml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/minimal.yml b/.github/workflows/minimal.yml index 2c0cd45ca9..2cc0213781 100644 --- a/.github/workflows/minimal.yml +++ b/.github/workflows/minimal.yml @@ -15,7 +15,7 @@ jobs: steps: - uses: actions/checkout@v4 - name: Setup Miniconda - uses: conda-incubator/setup-miniconda@v2.3.0 + uses: conda-incubator/setup-miniconda@v3.0.1 with: channels: conda-forge environment-file: environment.yml diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index aa7158f1cf..0c3c49d78d 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -42,7 +42,7 @@ jobs: with: fetch-depth: 0 - name: Setup Miniconda - uses: conda-incubator/setup-miniconda@v2.3.0 + uses: conda-incubator/setup-miniconda@v3.0.1 with: channels: conda-forge python-version: ${{ matrix.python-version }} diff --git a/.github/workflows/windows-testing.yml b/.github/workflows/windows-testing.yml index 78945e97aa..eeee5b704d 100644 --- a/.github/workflows/windows-testing.yml +++ b/.github/workflows/windows-testing.yml @@ -21,7 +21,7 @@ jobs: - uses: actions/checkout@v4 with: fetch-depth: 0 - - uses: conda-incubator/setup-miniconda@v2.3.0 + - uses: conda-incubator/setup-miniconda@v3.0.1 with: auto-update-conda: true python-version: ${{ matrix.python-version }} From 25dbeeda7d3a300569b358c157c5bd1c02ddaec3 Mon Sep 17 00:00:00 2001 From: Janick Martinez Esturo Date: Tue, 5 Dec 2023 22:31:03 +0100 Subject: [PATCH 003/130] * Cache result of FSStore._fsspec_installed (#1581) Prevent runtime-overhead in doing this check multiple times --- docs/release.rst | 3 +++ zarr/storage.py | 2 ++ 2 files changed, 5 insertions(+) diff --git a/docs/release.rst b/docs/release.rst index 9873d62896..842c36e290 100644 --- a/docs/release.rst +++ b/docs/release.rst @@ -43,6 +43,9 @@ Docs Maintenance ~~~~~~~~~~~ +* Cache result of ``FSStore._fsspec_installed()``. + By :user:`Janick Martinez Esturo ` :issue:`1581`. + * Extend copyright notice to 2023. By :user:`Jack Kelly ` :issue:`1528`. diff --git a/zarr/storage.py b/zarr/storage.py index b36f804ebd..a7426e5345 100644 --- a/zarr/storage.py +++ b/zarr/storage.py @@ -28,6 +28,7 @@ import zipfile from collections import OrderedDict from collections.abc import MutableMapping +from functools import lru_cache from os import scandir from pickle import PicklingError from threading import Lock, RLock @@ -1540,6 +1541,7 @@ def clear(self): self.map.clear() @classmethod + @lru_cache(maxsize=None) def _fsspec_installed(cls): """Returns true if fsspec is installed""" import importlib.util From 8579e21c80927afbc26153153ca8eedc91a6ff6f Mon Sep 17 00:00:00 2001 From: David Stansby Date: Thu, 7 Dec 2023 21:15:55 +0000 Subject: [PATCH 004/130] Bump version of black in pre-commit (#1559) --- .pre-commit-config.yaml | 2 +- bench/compress_normal.py | 1 - zarr/_storage/absstore.py | 3 +- zarr/_storage/store.py | 1 - zarr/_storage/v3.py | 1 - zarr/attrs.py | 6 ---- zarr/convenience.py | 20 +++-------- zarr/creation.py | 2 -- zarr/hierarchy.py | 12 +++---- zarr/indexing.py | 25 -------------- zarr/meta.py | 1 - zarr/n5.py | 57 -------------------------------- zarr/storage.py | 5 --- zarr/tests/test_attrs.py | 6 ---- zarr/tests/test_convenience.py | 7 ---- zarr/tests/test_creation.py | 9 ----- zarr/tests/test_dim_separator.py | 1 - zarr/tests/test_filters.py | 12 ------- zarr/tests/test_hierarchy.py | 3 -- zarr/tests/test_indexing.py | 35 -------------------- zarr/tests/test_info.py | 1 - zarr/tests/test_meta.py | 19 ----------- zarr/tests/test_storage.py | 20 ----------- zarr/tests/test_storage_v3.py | 10 ------ zarr/tests/test_sync.py | 2 -- zarr/tests/test_util.py | 2 -- zarr/util.py | 6 ---- 27 files changed, 11 insertions(+), 258 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index f22dc39832..e985d24000 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -14,7 +14,7 @@ repos: # Respect `exclude` and `extend-exclude` settings. args: ["--force-exclude"] - repo: https://github.com/psf/black - rev: 22.12.0 + rev: 23.10.1 hooks: - id: black - repo: https://github.com/codespell-project/codespell diff --git a/bench/compress_normal.py b/bench/compress_normal.py index 9f1655541c..803d54b76b 100644 --- a/bench/compress_normal.py +++ b/bench/compress_normal.py @@ -8,7 +8,6 @@ from zarr import blosc if __name__ == "__main__": - sys.path.insert(0, "..") # setup diff --git a/zarr/_storage/absstore.py b/zarr/_storage/absstore.py index f62529f096..c9a113148c 100644 --- a/zarr/_storage/absstore.py +++ b/zarr/_storage/absstore.py @@ -87,7 +87,7 @@ def __init__( "https://{}.blob.core.windows.net/".format(account_name), container, credential=account_key, - **blob_service_kwargs + **blob_service_kwargs, ) self.client = client @@ -240,7 +240,6 @@ def __setitem__(self, key, value): super().__setitem__(key, value) def rmdir(self, path=None): - if not path: # Currently allowing clear to delete everything as in v2 diff --git a/zarr/_storage/store.py b/zarr/_storage/store.py index 8daedae48f..80e4ad8f75 100644 --- a/zarr/_storage/store.py +++ b/zarr/_storage/store.py @@ -629,7 +629,6 @@ def _rmdir_from_keys(store: StoreLike, path: Optional[str] = None) -> None: def _rmdir_from_keys_v3(store: StoreV3, path: str = "") -> None: - meta_dir = meta_root + path meta_dir = meta_dir.rstrip("/") _rmdir_from_keys(store, meta_dir) diff --git a/zarr/_storage/v3.py b/zarr/_storage/v3.py index 00dc085dac..32e78f7a34 100644 --- a/zarr/_storage/v3.py +++ b/zarr/_storage/v3.py @@ -118,7 +118,6 @@ def _get_files_and_dirs_from_path(store, path): class FSStoreV3(FSStore, StoreV3): - # FSStoreV3 doesn't use this (FSStore uses it within _normalize_key) _META_KEYS = () diff --git a/zarr/attrs.py b/zarr/attrs.py index 01fc617b3c..e967c5b853 100644 --- a/zarr/attrs.py +++ b/zarr/attrs.py @@ -26,7 +26,6 @@ class Attributes(MutableMapping): """ def __init__(self, store, key=".zattrs", read_only=False, cache=True, synchronizer=None): - self._version = getattr(store, "_store_version", 2) _Store = Store if self._version == 2 else StoreV3 self.store = _Store._ensure_store(store) @@ -73,7 +72,6 @@ def __getitem__(self, item): return self.asdict()[item] def _write_op(self, f, *args, **kwargs): - # guard condition if self.read_only: raise PermissionError("attributes are read-only") @@ -89,7 +87,6 @@ def __setitem__(self, item, value): self._write_op(self._setitem_nosync, item, value) def _setitem_nosync(self, item, value): - # load existing data d = self._get_nosync() @@ -106,7 +103,6 @@ def __delitem__(self, item): self._write_op(self._delitem_nosync, item) def _delitem_nosync(self, key): - # load existing data d = self._get_nosync() @@ -128,7 +124,6 @@ def put(self, d): self._write_op(self._put_nosync, dict(attributes=d)) def _put_nosync(self, d): - d_to_check = d if self._version == 2 else d["attributes"] if not all(isinstance(item, str) for item in d_to_check): # TODO: Raise an error for non-string keys @@ -178,7 +173,6 @@ def update(self, *args, **kwargs): self._write_op(self._update_nosync, *args, **kwargs) def _update_nosync(self, *args, **kwargs): - # load existing data d = self._get_nosync() diff --git a/zarr/convenience.py b/zarr/convenience.py index 0ee8a8d323..9c0deeea47 100644 --- a/zarr/convenience.py +++ b/zarr/convenience.py @@ -675,10 +675,8 @@ def copy_store( # setup logging with _LogWriter(log) as log: - # iterate over source keys for source_key in sorted(source.keys()): - # filter to keys under source path if source_store_version == 2: if not source_key.startswith(source_path): @@ -757,7 +755,7 @@ def copy( log=None, if_exists="raise", dry_run=False, - **create_kws + **create_kws, ): """Copy the `source` array or group into the `dest` group. @@ -878,7 +876,6 @@ def copy( # setup logging with _LogWriter(log) as log: - # do the copying n_copied, n_skipped, n_bytes_copied = _copy( log, @@ -890,7 +887,7 @@ def copy( without_attrs=without_attrs, if_exists=if_exists, dry_run=dry_run, - **create_kws + **create_kws, ) # log a final message with a summary of what happened @@ -948,12 +945,10 @@ def _copy(log, source, dest, name, root, shallow, without_attrs, if_exists, dry_ # take action if do_copy: - # log a message about what we're going to do log("copy {} {} {}".format(source.name, source.shape, source.dtype)) if not dry_run: - # clear the way if exists: del dest[name] @@ -1038,12 +1033,10 @@ def _copy(log, source, dest, name, root, shallow, without_attrs, if_exists, dry_ # take action if do_copy: - # log action log("copy {}".format(source.name)) if not dry_run: - # clear the way if exists_array: del dest[name] @@ -1056,7 +1049,6 @@ def _copy(log, source, dest, name, root, shallow, without_attrs, if_exists, dry_ grp.attrs.update(source.attrs) else: - # setup for dry run without creating any groups in the # destination if dest is not None: @@ -1076,7 +1068,7 @@ def _copy(log, source, dest, name, root, shallow, without_attrs, if_exists, dry_ without_attrs=without_attrs, if_exists=if_exists, dry_run=dry_run, - **create_kws + **create_kws, ) n_copied += c n_skipped += s @@ -1099,7 +1091,7 @@ def copy_all( log=None, if_exists="raise", dry_run=False, - **create_kws + **create_kws, ): """Copy all children of the `source` group into the `dest` group. @@ -1189,7 +1181,6 @@ def copy_all( # setup logging with _LogWriter(log) as log: - for k in source.keys(): c, s, b = _copy( log, @@ -1201,7 +1192,7 @@ def copy_all( without_attrs=without_attrs, if_exists=if_exists, dry_run=dry_run, - **create_kws + **create_kws, ) n_copied += c n_skipped += s @@ -1262,7 +1253,6 @@ def is_zarr_key(key): return key.endswith(".zarray") or key.endswith(".zgroup") or key.endswith(".zattrs") else: - assert_zarr_v3_api_available() sfx = _get_metadata_suffix(store) # type: ignore diff --git a/zarr/creation.py b/zarr/creation.py index 726d0b5932..6227f90b7b 100644 --- a/zarr/creation.py +++ b/zarr/creation.py @@ -234,7 +234,6 @@ def create( def _kwargs_compat(compressor, fill_value, kwargs): - # to be compatible with h5py, as well as backwards-compatible with Zarr # 1.x, accept 'compression' and 'compression_opts' keyword arguments @@ -697,7 +696,6 @@ def open_array( def _like_args(a, kwargs): - shape, chunks = _get_shape_chunks(a) if shape is not None: kwargs.setdefault("shape", shape) diff --git a/zarr/hierarchy.py b/zarr/hierarchy.py index 3361969f08..1cfea89c81 100644 --- a/zarr/hierarchy.py +++ b/zarr/hierarchy.py @@ -145,7 +145,7 @@ def __init__( synchronizer=None, zarr_version=None, *, - meta_array=None + meta_array=None, ): store: BaseStore = _normalize_store_arg(store, zarr_version=zarr_version) if zarr_version is None: @@ -919,7 +919,6 @@ def tree(self, expand=False, level=None): return TreeViewer(self, expand=expand, level=level) def _write_op(self, f, *args, **kwargs): - # guard condition if self._read_only: raise ReadOnlyError() @@ -1094,7 +1093,6 @@ def create_dataset(self, name, **kwargs): return self._write_op(self._create_dataset_nosync, name, **kwargs) def _create_dataset_nosync(self, name, data=None, **kwargs): - assert "mode" not in kwargs path = self._item_path(name) @@ -1138,11 +1136,9 @@ def require_dataset(self, name, shape, dtype=None, exact=False, **kwargs): ) def _require_dataset_nosync(self, name, shape, dtype=None, exact=False, **kwargs): - path = self._item_path(name) if contains_array(self._store, path): - # array already exists at path, validate that it is the right shape and type synchronizer = kwargs.get("synchronizer", self._synchronizer) @@ -1235,7 +1231,7 @@ def _full_nosync(self, name, fill_value, **kwargs): path=path, chunk_store=self._chunk_store, fill_value=fill_value, - **kwargs + **kwargs, ) def array(self, name, data, **kwargs): @@ -1361,7 +1357,7 @@ def group( path=None, *, zarr_version=None, - meta_array=None + meta_array=None, ): """Create a group. @@ -1452,7 +1448,7 @@ def open_group( storage_options=None, *, zarr_version=None, - meta_array=None + meta_array=None, ): """Open a group using file-mode-like semantics. diff --git a/zarr/indexing.py b/zarr/indexing.py index 487cc8b9d9..3042147ebb 100644 --- a/zarr/indexing.py +++ b/zarr/indexing.py @@ -111,7 +111,6 @@ def is_pure_orthogonal_indexing(selection, ndim): def normalize_integer_selection(dim_sel, dim_len): - # normalize type to int dim_sel = int(dim_sel) @@ -145,7 +144,6 @@ def normalize_integer_selection(dim_sel, dim_len): class IntDimIndexer: def __init__(self, dim_sel, dim_len, dim_chunk_len): - # normalize dim_sel = normalize_integer_selection(dim_sel, dim_len) @@ -169,7 +167,6 @@ def ceildiv(a, b): class SliceDimIndexer: def __init__(self, dim_sel, dim_len, dim_chunk_len): - # normalize self.start, self.stop, self.step = dim_sel.indices(dim_len) if self.step < 1: @@ -182,14 +179,12 @@ def __init__(self, dim_sel, dim_len, dim_chunk_len): self.nchunks = ceildiv(self.dim_len, self.dim_chunk_len) def __iter__(self): - # figure out the range of chunks we need to visit dim_chunk_ix_from = self.start // self.dim_chunk_len dim_chunk_ix_to = ceildiv(self.stop, self.dim_chunk_len) # iterate over chunks in range for dim_chunk_ix in range(dim_chunk_ix_from, dim_chunk_ix_to): - # compute offsets for chunk within overall array dim_offset = dim_chunk_ix * self.dim_chunk_len dim_limit = min(self.dim_len, (dim_chunk_ix + 1) * self.dim_chunk_len) @@ -237,7 +232,6 @@ def check_selection_length(selection, shape): def replace_ellipsis(selection, shape): - selection = ensure_tuple(selection) # count number of ellipsis present @@ -330,14 +324,12 @@ def is_basic_selection(selection): # noinspection PyProtectedMember class BasicIndexer: def __init__(self, selection, array): - # handle ellipsis selection = replace_ellipsis(selection, array._shape) # setup per-dimension indexers dim_indexers = [] for dim_sel, dim_len, dim_chunk_len in zip(selection, array._shape, array._chunks): - if is_integer(dim_sel): dim_indexer = IntDimIndexer(dim_sel, dim_len, dim_chunk_len) @@ -358,7 +350,6 @@ def __init__(self, selection, array): def __iter__(self): for dim_projections in itertools.product(*self.dim_indexers): - chunk_coords = tuple(p.dim_chunk_ix for p in dim_projections) chunk_selection = tuple(p.dim_chunk_sel for p in dim_projections) out_selection = tuple( @@ -370,7 +361,6 @@ def __iter__(self): class BoolArrayDimIndexer: def __init__(self, dim_sel, dim_len, dim_chunk_len): - # check number of dimensions if not is_bool_array(dim_sel, 1): raise IndexError( @@ -402,10 +392,8 @@ def __init__(self, dim_sel, dim_len, dim_chunk_len): self.dim_chunk_ixs = np.nonzero(self.chunk_nitems)[0] def __iter__(self): - # iterate over chunks with at least one item for dim_chunk_ix in self.dim_chunk_ixs: - # find region in chunk dim_offset = dim_chunk_ix * self.dim_chunk_len dim_chunk_sel = self.dim_sel[dim_offset : dim_offset + self.dim_chunk_len] @@ -472,7 +460,6 @@ def __init__( boundscheck=True, order=Order.UNKNOWN, ): - # ensure 1d array dim_sel = np.asanyarray(dim_sel) if not is_integer_array(dim_sel, 1): @@ -526,9 +513,7 @@ def __init__( self.chunk_nitems_cumsum = np.cumsum(self.chunk_nitems) def __iter__(self): - for dim_chunk_ix in self.dim_chunk_ixs: - # find region in output if dim_chunk_ix == 0: start = 0 @@ -602,7 +587,6 @@ def oindex_set(a, selection, value): # noinspection PyProtectedMember class OrthogonalIndexer: def __init__(self, selection, array): - # handle ellipsis selection = replace_ellipsis(selection, array._shape) @@ -612,7 +596,6 @@ def __init__(self, selection, array): # setup per-dimension indexers dim_indexers = [] for dim_sel, dim_len, dim_chunk_len in zip(selection, array._shape, array._chunks): - if is_integer(dim_sel): dim_indexer = IntDimIndexer(dim_sel, dim_len, dim_chunk_len) @@ -649,7 +632,6 @@ def __init__(self, selection, array): def __iter__(self): for dim_projections in itertools.product(*self.dim_indexers): - chunk_coords = tuple(p.dim_chunk_ix for p in dim_projections) chunk_selection = tuple(p.dim_chunk_sel for p in dim_projections) out_selection = tuple( @@ -658,7 +640,6 @@ def __iter__(self): # handle advanced indexing arrays orthogonally if self.is_advanced: - # N.B., numpy doesn't support orthogonal indexing directly as yet, # so need to work around via np.ix_. Also np.ix_ does not support a # mixture of arrays and slices or integers, so need to convert slices @@ -692,7 +673,6 @@ def __setitem__(self, selection, value): # noinspection PyProtectedMember class BlockIndexer: def __init__(self, selection, array): - # handle ellipsis selection = replace_ellipsis(selection, array._shape) @@ -794,7 +774,6 @@ def is_mask_selection(selection, array): # noinspection PyProtectedMember class CoordinateIndexer: def __init__(self, selection, array): - # some initial normalization selection = ensure_tuple(selection) selection = tuple([i] if is_integer(i) else i for i in selection) @@ -810,7 +789,6 @@ def __init__(self, selection, array): # handle wraparound, boundscheck for dim_sel, dim_len in zip(selection, array.shape): - # handle wraparound wraparound_indices(dim_sel, dim_len) @@ -861,10 +839,8 @@ def __init__(self, selection, array): self.chunk_mixs = np.unravel_index(self.chunk_rixs, array._cdata_shape) def __iter__(self): - # iterate over chunks for i, chunk_rix in enumerate(self.chunk_rixs): - chunk_coords = tuple(m[i] for m in self.chunk_mixs) if chunk_rix == 0: start = 0 @@ -891,7 +867,6 @@ def __iter__(self): # noinspection PyProtectedMember class MaskIndexer(CoordinateIndexer): def __init__(self, selection, array): - # some initial normalization selection = ensure_tuple(selection) selection = replace_lists(selection) diff --git a/zarr/meta.py b/zarr/meta.py index 48791ddf17..f23889f3ea 100644 --- a/zarr/meta.py +++ b/zarr/meta.py @@ -89,7 +89,6 @@ class Metadata2: @classmethod def parse_metadata(cls, s: Union[MappingType, bytes, str]) -> MappingType[str, Any]: - # Here we allow that a store may return an already-parsed metadata object, # or a string of JSON that we will parse here. We allow for an already-parsed # object to accommodate a consolidated metadata store, where all the metadata for diff --git a/zarr/n5.py b/zarr/n5.py index 7e73905527..44b44e69e2 100644 --- a/zarr/n5.py +++ b/zarr/n5.py @@ -72,21 +72,18 @@ class N5Store(NestedDirectoryStore): def __getitem__(self, key: str) -> bytes: if key.endswith(zarr_group_meta_key): - key_new = key.replace(zarr_group_meta_key, n5_attrs_key) value = group_metadata_to_zarr(self._load_n5_attrs(key_new)) return json_dumps(value) elif key.endswith(zarr_array_meta_key): - key_new = key.replace(zarr_array_meta_key, n5_attrs_key) top_level = key == zarr_array_meta_key value = array_metadata_to_zarr(self._load_n5_attrs(key_new), top_level=top_level) return json_dumps(value) elif key.endswith(zarr_attrs_key): - key_new = key.replace(zarr_attrs_key, n5_attrs_key) value = attrs_to_zarr(self._load_n5_attrs(key_new)) @@ -104,9 +101,7 @@ def __getitem__(self, key: str) -> bytes: return super().__getitem__(key_new) def __setitem__(self, key: str, value: Any): - if key.endswith(zarr_group_meta_key): - key_new = key.replace(zarr_group_meta_key, n5_attrs_key) n5_attrs = self._load_n5_attrs(key_new) @@ -115,7 +110,6 @@ def __setitem__(self, key: str, value: Any): value = json_dumps(n5_attrs) elif key.endswith(zarr_array_meta_key): - key_new = key.replace(zarr_array_meta_key, n5_attrs_key) top_level = key == zarr_array_meta_key n5_attrs = self._load_n5_attrs(key_new) @@ -123,7 +117,6 @@ def __setitem__(self, key: str, value: Any): value = json_dumps(n5_attrs) elif key.endswith(zarr_attrs_key): - key_new = key.replace(zarr_attrs_key, n5_attrs_key) n5_attrs = self._load_n5_attrs(key_new) @@ -166,9 +159,7 @@ def __delitem__(self, key: str): super().__delitem__(key_new) def __contains__(self, key): - if key.endswith(zarr_group_meta_key): - key_new = key.replace(zarr_group_meta_key, n5_attrs_key) if key_new not in self: return False @@ -176,18 +167,15 @@ def __contains__(self, key): return "dimensions" not in self._load_n5_attrs(key_new) elif key.endswith(zarr_array_meta_key): - key_new = key.replace(zarr_array_meta_key, n5_attrs_key) # array if attributes contain 'dimensions' return "dimensions" in self._load_n5_attrs(key_new) elif key.endswith(zarr_attrs_key): - key_new = key.replace(zarr_attrs_key, n5_attrs_key) return self._contains_attrs(key_new) elif is_chunk_key(key): - key_new = invert_chunk_coords(key) else: key_new = key @@ -198,7 +186,6 @@ def __eq__(self, other): return isinstance(other, N5Store) and self.path == other.path def listdir(self, path: Optional[str] = None): - if path is not None: path = invert_chunk_coords(path) path = cast(str, path) @@ -208,7 +195,6 @@ def listdir(self, path: Optional[str] = None): children = super().listdir(path=path) if self._is_array(path): - # replace n5 attribute file with respective zarr attribute files children.remove(n5_attrs_key) children.append(zarr_array_meta_key) @@ -234,7 +220,6 @@ def listdir(self, path: Optional[str] = None): return sorted(new_children) elif self._is_group(path): - # replace n5 attribute file with respective zarr attribute files children.remove(n5_attrs_key) children.append(zarr_group_meta_key) @@ -244,7 +229,6 @@ def listdir(self, path: Optional[str] = None): return sorted(children) else: - return children def _load_n5_attrs(self, path: str) -> Dict[str, Any]: @@ -255,7 +239,6 @@ def _load_n5_attrs(self, path: str) -> Dict[str, Any]: return {} def _is_group(self, path: str): - if path is None: attrs_key = n5_attrs_key else: @@ -265,7 +248,6 @@ def _is_group(self, path: str): return len(n5_attrs) > 0 and "dimensions" not in n5_attrs def _is_array(self, path: str): - if path is None: attrs_key = n5_attrs_key else: @@ -274,7 +256,6 @@ def _is_array(self, path: str): return "dimensions" in self._load_n5_attrs(attrs_key) def _contains_attrs(self, path: str): - if path is None: attrs_key = n5_attrs_key else: @@ -376,21 +357,18 @@ def _normalize_key(self, key: str): def __getitem__(self, key: str) -> bytes: if key.endswith(zarr_group_meta_key): - key_new = key.replace(zarr_group_meta_key, self._group_meta_key) value = group_metadata_to_zarr(self._load_n5_attrs(key_new)) return json_dumps(value) elif key.endswith(zarr_array_meta_key): - key_new = key.replace(zarr_array_meta_key, self._array_meta_key) top_level = key == zarr_array_meta_key value = array_metadata_to_zarr(self._load_n5_attrs(key_new), top_level=top_level) return json_dumps(value) elif key.endswith(zarr_attrs_key): - key_new = key.replace(zarr_attrs_key, self._attrs_key) value = attrs_to_zarr(self._load_n5_attrs(key_new)) @@ -409,7 +387,6 @@ def __getitem__(self, key: str) -> bytes: def __setitem__(self, key: str, value: Any): if key.endswith(zarr_group_meta_key): - key_new = key.replace(zarr_group_meta_key, self._group_meta_key) n5_attrs = self._load_n5_attrs(key_new) @@ -418,7 +395,6 @@ def __setitem__(self, key: str, value: Any): value = json_dumps(n5_attrs) elif key.endswith(zarr_array_meta_key): - key_new = key.replace(zarr_array_meta_key, self._array_meta_key) top_level = key == zarr_array_meta_key n5_attrs = self._load_n5_attrs(key_new) @@ -427,7 +403,6 @@ def __setitem__(self, key: str, value: Any): value = json_dumps(n5_attrs) elif key.endswith(zarr_attrs_key): - key_new = key.replace(zarr_attrs_key, self._attrs_key) n5_attrs = self._load_n5_attrs(key_new) @@ -456,7 +431,6 @@ def __setitem__(self, key: str, value: Any): super().__setitem__(key_new, value) def __delitem__(self, key: str): - if key.endswith(zarr_group_meta_key): key_new = key.replace(zarr_group_meta_key, self._group_meta_key) elif key.endswith(zarr_array_meta_key): @@ -471,7 +445,6 @@ def __delitem__(self, key: str): def __contains__(self, key: Any): if key.endswith(zarr_group_meta_key): - key_new = key.replace(zarr_group_meta_key, self._group_meta_key) if key_new not in self: return False @@ -479,13 +452,11 @@ def __contains__(self, key: Any): return "dimensions" not in self._load_n5_attrs(key_new) elif key.endswith(zarr_array_meta_key): - key_new = key.replace(zarr_array_meta_key, self._array_meta_key) # array if attributes contain 'dimensions' return "dimensions" in self._load_n5_attrs(key_new) elif key.endswith(zarr_attrs_key): - key_new = key.replace(zarr_attrs_key, self._attrs_key) return self._contains_attrs(key_new) @@ -508,7 +479,6 @@ def listdir(self, path: Optional[str] = None): # doesn't provide. children = super().listdir(path=path) if self._is_array(path): - # replace n5 attribute file with respective zarr attribute files children.remove(self._array_meta_key) children.append(zarr_array_meta_key) @@ -532,7 +502,6 @@ def listdir(self, path: Optional[str] = None): return sorted(new_children) elif self._is_group(path): - # replace n5 attribute file with respective zarr attribute files children.remove(self._group_meta_key) children.append(zarr_group_meta_key) @@ -550,7 +519,6 @@ def _load_n5_attrs(self, path: str): return {} def _is_group(self, path: Optional[str]): - if path is None: attrs_key = self._attrs_key else: @@ -560,7 +528,6 @@ def _is_group(self, path: Optional[str]): return len(n5_attrs) > 0 and "dimensions" not in n5_attrs def _is_array(self, path: Optional[str]): - if path is None: attrs_key = self._attrs_key else: @@ -569,7 +536,6 @@ def _is_array(self, path: Optional[str]): return "dimensions" in self._load_n5_attrs(attrs_key) def _contains_attrs(self, path: Optional[str]): - if path is None: attrs_key = self._attrs_key else: @@ -712,7 +678,6 @@ def attrs_to_zarr(attrs: Dict[str, Any]) -> Dict[str, Any]: def compressor_config_to_n5(compressor_config: Optional[Dict[str, Any]]) -> Dict[str, Any]: - if compressor_config is None: return {"type": "raw"} else: @@ -726,19 +691,16 @@ def compressor_config_to_n5(compressor_config: Optional[Dict[str, Any]]) -> Dict n5_config = {"type": codec_id} if codec_id == "bz2": - n5_config["type"] = "bzip2" n5_config["blockSize"] = _compressor_config["level"] elif codec_id == "blosc": - n5_config["cname"] = _compressor_config["cname"] n5_config["clevel"] = _compressor_config["clevel"] n5_config["shuffle"] = _compressor_config["shuffle"] n5_config["blocksize"] = _compressor_config["blocksize"] elif codec_id == "lzma": - # Switch to XZ for N5 if we are using the default XZ format. # Note: 4 is the default, which is lzma.CHECK_CRC64. if _compressor_config["format"] == 1 and _compressor_config["check"] in [-1, 4]: @@ -760,50 +722,42 @@ def compressor_config_to_n5(compressor_config: Optional[Dict[str, Any]]) -> Dict n5_config["preset"] = 6 elif codec_id == "zlib": - n5_config["type"] = "gzip" n5_config["level"] = _compressor_config["level"] n5_config["useZlib"] = True elif codec_id == "gzip": - n5_config["type"] = "gzip" n5_config["level"] = _compressor_config["level"] n5_config["useZlib"] = False else: - n5_config.update({k: v for k, v in _compressor_config.items() if k != "type"}) return n5_config def compressor_config_to_zarr(compressor_config: Dict[str, Any]) -> Optional[Dict[str, Any]]: - codec_id = compressor_config["type"] zarr_config = {"id": codec_id} if codec_id == "bzip2": - zarr_config["id"] = "bz2" zarr_config["level"] = compressor_config["blockSize"] elif codec_id == "blosc": - zarr_config["cname"] = compressor_config["cname"] zarr_config["clevel"] = compressor_config["clevel"] zarr_config["shuffle"] = compressor_config["shuffle"] zarr_config["blocksize"] = compressor_config["blocksize"] elif codec_id == "lzma": - zarr_config["format"] = compressor_config["format"] zarr_config["check"] = compressor_config["check"] zarr_config["preset"] = compressor_config["preset"] zarr_config["filters"] = compressor_config["filters"] elif codec_id == "xz": - zarr_config["id"] = "lzma" zarr_config["format"] = 1 # lzma.FORMAT_XZ zarr_config["check"] = -1 @@ -811,7 +765,6 @@ def compressor_config_to_zarr(compressor_config: Dict[str, Any]) -> Optional[Dic zarr_config["filters"] = None elif codec_id == "gzip": - if "useZlib" in compressor_config and compressor_config["useZlib"]: zarr_config["id"] = "zlib" zarr_config["level"] = compressor_config["level"] @@ -820,22 +773,18 @@ def compressor_config_to_zarr(compressor_config: Dict[str, Any]) -> Optional[Dic zarr_config["level"] = compressor_config["level"] elif codec_id == "raw": - return None else: - zarr_config.update({k: v for k, v in compressor_config.items() if k != "type"}) return zarr_config class N5ChunkWrapper(Codec): - codec_id = "n5_wrapper" def __init__(self, dtype, chunk_shape, compressor_config=None, compressor=None): - self.dtype = np.dtype(dtype) self.chunk_shape = tuple(chunk_shape) # is the dtype a little endian format? @@ -860,7 +809,6 @@ def get_config(self): return config def encode(self, chunk): - assert chunk.flags.c_contiguous header = self._create_header(chunk) @@ -872,12 +820,10 @@ def encode(self, chunk): return header + chunk.tobytes(order="A") def decode(self, chunk, out=None) -> bytes: - len_header, chunk_shape = self._read_header(chunk) chunk = chunk[len_header:] if out is not None: - # out should only be used if we read a complete chunk assert chunk_shape == self.chunk_shape, "Expected chunk of shape {}, found {}".format( self.chunk_shape, chunk_shape @@ -895,7 +841,6 @@ def decode(self, chunk, out=None) -> bytes: return out else: - if self._compressor: chunk = self._compressor.decode(chunk) @@ -915,7 +860,6 @@ def decode(self, chunk, out=None) -> bytes: @staticmethod def _create_header(chunk): - mode = struct.pack(">H", 0) num_dims = struct.pack(">H", len(chunk.shape)) shape = b"".join(struct.pack(">I", d) for d in chunk.shape[::-1]) @@ -924,7 +868,6 @@ def _create_header(chunk): @staticmethod def _read_header(chunk): - num_dims = struct.unpack(">H", chunk[2:4])[0] shape = tuple( struct.unpack(">I", chunk[i : i + 4])[0] for i in range(4, num_dims * 4 + 4, 4) diff --git a/zarr/storage.py b/zarr/storage.py index a7426e5345..585417f59c 100644 --- a/zarr/storage.py +++ b/zarr/storage.py @@ -483,7 +483,6 @@ def _init_array_metadata( dimension_separator=None, storage_transformers=(), ): - store_version = getattr(store, "_store_version", 2) path = normalize_storage_path(path) @@ -688,7 +687,6 @@ def _init_group_metadata( path: Optional[str] = None, chunk_store: Optional[StoreLike] = None, ): - store_version = getattr(store, "_store_version", 2) path = normalize_storage_path(path) @@ -1056,7 +1054,6 @@ class DirectoryStore(Store): """ def __init__(self, path, normalize_keys=False, dimension_separator=None): - # guard conditions path = os.path.abspath(path) if os.path.exists(path) and not os.path.isdir(path): @@ -1416,7 +1413,6 @@ def _normalize_key(self, key): def getitems( self, keys: Sequence[str], *, contexts: Mapping[str, Context] ) -> Mapping[str, Any]: - keys_transformed = [self._normalize_key(key) for key in keys] results = self.map.getitems(keys_transformed, on_error="omit") # The function calling this method may not recognize the transformed keys @@ -1770,7 +1766,6 @@ def __init__( mode="a", dimension_separator=None, ): - # store properties path = os.path.abspath(path) self.path = path diff --git a/zarr/tests/test_attrs.py b/zarr/tests/test_attrs.py index 7dd5b340a2..2d9553971b 100644 --- a/zarr/tests/test_attrs.py +++ b/zarr/tests/test_attrs.py @@ -30,7 +30,6 @@ def init_attributes(self, store, read_only=False, cache=True, zarr_version=2): return Attributes(store, key=root + "attrs", read_only=read_only, cache=cache) def test_storage(self, zarr_version): - store = _init_store(zarr_version) root = ".z" if zarr_version == 2 else meta_root attrs_key = root + "attrs" @@ -50,7 +49,6 @@ def test_storage(self, zarr_version): assert dict(foo="bar", baz=42) == d def test_utf8_encoding(self, zarr_version): - project_root = pathlib.Path(zarr.__file__).resolve().parent.parent fixdir = project_root / "fixture" testdir = fixdir / "utf8attrs" @@ -67,7 +65,6 @@ def test_utf8_encoding(self, zarr_version): assert fixture["utf8attrs"].attrs.asdict() == dict(foo="た") def test_get_set_del_contains(self, zarr_version): - store = _init_store(zarr_version) a = self.init_attributes(store, zarr_version=zarr_version) assert "foo" not in a @@ -84,7 +81,6 @@ def test_get_set_del_contains(self, zarr_version): a["foo"] def test_update_put(self, zarr_version): - store = _init_store(zarr_version) a = self.init_attributes(store, zarr_version=zarr_version) assert "foo" not in a @@ -102,7 +98,6 @@ def test_update_put(self, zarr_version): assert "baz" not in a def test_iterators(self, zarr_version): - store = _init_store(zarr_version) a = self.init_attributes(store, zarr_version=zarr_version) assert 0 == len(a) @@ -232,7 +227,6 @@ def test_caching_on(self, zarr_version): assert get_cnt == store.counter["__getitem__", attrs_key] def test_caching_off(self, zarr_version): - # setup store store = CountingDict() if zarr_version == 2 else CountingDictV3() attrs_key = ".zattrs" if zarr_version == 2 else "meta/root/attrs" diff --git a/zarr/tests/test_convenience.py b/zarr/tests/test_convenience.py index 389ce90a9d..7d190adc2c 100644 --- a/zarr/tests/test_convenience.py +++ b/zarr/tests/test_convenience.py @@ -57,7 +57,6 @@ def _init_creation_kwargs(zarr_version): @pytest.mark.parametrize("zarr_version", _VERSIONS) def test_open_array(path_type, zarr_version): - store = tempfile.mkdtemp() atexit.register(atexit_rmtree, store) store = path_type(store) @@ -86,7 +85,6 @@ def test_open_array(path_type, zarr_version): @pytest.mark.parametrize("zarr_version", _VERSIONS) def test_open_group(path_type, zarr_version): - store = tempfile.mkdtemp() atexit.register(atexit_rmtree, store) store = path_type(store) @@ -210,7 +208,6 @@ def test_tree(zarr_version): def test_consolidate_metadata( with_chunk_store, zarr_version, listable, monkeypatch, stores_from_path ): - # setup initial data if stores_from_path: store = tempfile.mkdtemp() @@ -399,7 +396,6 @@ def test_save_array_separator(tmpdir, options): class TestCopyStore(unittest.TestCase): - _version = 2 def setUp(self): @@ -536,7 +532,6 @@ def test_if_exists(self): @pytest.mark.skipif(not v3_api_available, reason="V3 is disabled") class TestCopyStoreV3(TestCopyStore): - _version = 3 def setUp(self): @@ -557,7 +552,6 @@ def test_mismatched_store_versions(self): def check_copied_array(original, copied, without_attrs=False, expect_props=None): - # setup source_h5py = original.__module__.startswith("h5py.") dest_h5py = copied.__module__.startswith("h5py.") @@ -621,7 +615,6 @@ def check_copied_array(original, copied, without_attrs=False, expect_props=None) def check_copied_group(original, copied, without_attrs=False, expect_props=None, shallow=False): - # setup if expect_props is None: expect_props = dict() diff --git a/zarr/tests/test_creation.py b/zarr/tests/test_creation.py index b44c6379fd..8e586abfff 100644 --- a/zarr/tests/test_creation.py +++ b/zarr/tests/test_creation.py @@ -74,7 +74,6 @@ def _init_creation_kwargs(zarr_version, at_root=True): @pytest.mark.parametrize("zarr_version", _VERSIONS) @pytest.mark.parametrize("at_root", [False, True]) def test_array(zarr_version, at_root): - expected_zarr_version = DEFAULT_ZARR_VERSION if zarr_version is None else zarr_version kwargs = _init_creation_kwargs(zarr_version, at_root) @@ -213,7 +212,6 @@ def test_full_additional_dtypes(zarr_version): @pytest.mark.parametrize("zarr_version", _VERSIONS) @pytest.mark.parametrize("at_root", [False, True]) def test_open_array(zarr_version, at_root, dimension_separator): - store = "data/array.zarr" kwargs = _init_creation_kwargs(zarr_version, at_root) @@ -329,7 +327,6 @@ def test_open_array(zarr_version, at_root, dimension_separator): def test_open_array_none(): - # open with both store and zarr_version = None z = open_array(mode="w", shape=100, chunks=10) assert isinstance(z, Array) @@ -339,7 +336,6 @@ def test_open_array_none(): @pytest.mark.parametrize("dimension_separator", [".", "/", None]) @pytest.mark.parametrize("zarr_version", _VERSIONS2) def test_open_array_infer_separator_from_store(zarr_version, dimension_separator): - if zarr_version == 3: StoreClass = DirectoryStoreV3 path = "data" @@ -370,7 +366,6 @@ def test_open_array_infer_separator_from_store(zarr_version, dimension_separator # TODO: N5 support for v3 @pytest.mark.parametrize("zarr_version", [None, 2]) def test_open_array_n5(zarr_version): - store = "data/array.zarr" kwargs = _init_creation_kwargs(zarr_version) @@ -409,7 +404,6 @@ def test_open_array_n5(zarr_version): @pytest.mark.parametrize("zarr_version", _VERSIONS) @pytest.mark.parametrize("at_root", [False, True]) def test_open_array_dict_store(zarr_version, at_root): - # dict will become a KVStore store = dict() kwargs = _init_creation_kwargs(zarr_version, at_root) @@ -503,7 +497,6 @@ def test_empty_like(zarr_version, at_root): @pytest.mark.parametrize("zarr_version", _VERSIONS) @pytest.mark.parametrize("at_root", [False, True]) def test_zeros_like(zarr_version, at_root): - kwargs = _init_creation_kwargs(zarr_version, at_root) expected_zarr_version = DEFAULT_ZARR_VERSION if zarr_version is None else zarr_version @@ -529,7 +522,6 @@ def test_zeros_like(zarr_version, at_root): @pytest.mark.parametrize("zarr_version", _VERSIONS) @pytest.mark.parametrize("at_root", [False, True]) def test_ones_like(zarr_version, at_root): - kwargs = _init_creation_kwargs(zarr_version, at_root) expected_zarr_version = DEFAULT_ZARR_VERSION if zarr_version is None else zarr_version @@ -556,7 +548,6 @@ def test_ones_like(zarr_version, at_root): @pytest.mark.parametrize("zarr_version", _VERSIONS) @pytest.mark.parametrize("at_root", [False, True]) def test_full_like(zarr_version, at_root): - kwargs = _init_creation_kwargs(zarr_version, at_root) expected_zarr_version = DEFAULT_ZARR_VERSION if zarr_version is None else zarr_version diff --git a/zarr/tests/test_dim_separator.py b/zarr/tests/test_dim_separator.py index 987852dfd0..0a5814e65f 100644 --- a/zarr/tests/test_dim_separator.py +++ b/zarr/tests/test_dim_separator.py @@ -46,7 +46,6 @@ def dataset(tmpdir, request): static = project_root / "fixture" / suffix if not static.exists(): # pragma: no cover - if "nested" in which: # No way to reproduce the nested_legacy file via code generator = NestedDirectoryStore diff --git a/zarr/tests/test_filters.py b/zarr/tests/test_filters.py index d55be9145f..fc63cdca8d 100644 --- a/zarr/tests/test_filters.py +++ b/zarr/tests/test_filters.py @@ -30,7 +30,6 @@ def test_array_with_delta_filter(): - # setup astype = "u1" dtype = "i8" @@ -38,7 +37,6 @@ def test_array_with_delta_filter(): data = np.arange(100, dtype=dtype) for compressor in compressors: - a = array(data, chunks=10, compressor=compressor, filters=filters) # check round-trip @@ -57,7 +55,6 @@ def test_array_with_delta_filter(): def test_array_with_astype_filter(): - # setup encode_dtype = "i1" decode_dtype = "i8" @@ -68,7 +65,6 @@ def test_array_with_astype_filter(): data = np.arange(shape, dtype=decode_dtype) for compressor in compressors: - a = array(data, chunks=chunks, compressor=compressor, filters=filters) # check round-trip @@ -88,7 +84,6 @@ def test_array_with_astype_filter(): def test_array_with_scaleoffset_filter(): - # setup astype = "u1" dtype = "f8" @@ -97,7 +92,6 @@ def test_array_with_scaleoffset_filter(): data = np.linspace(1000, 1001, 34, dtype="f8") for compressor in compressors: - a = array(data, chunks=5, compressor=compressor, filters=filters) # check round-trip @@ -116,7 +110,6 @@ def test_array_with_scaleoffset_filter(): def test_array_with_quantize_filter(): - # setup dtype = "f8" digits = 3 @@ -125,7 +118,6 @@ def test_array_with_quantize_filter(): data = np.linspace(0, 1, 34, dtype=dtype) for compressor in compressors: - a = array(data, chunks=5, compressor=compressor, filters=filters) # check round-trip @@ -144,14 +136,12 @@ def test_array_with_quantize_filter(): def test_array_with_packbits_filter(): - # setup flt = PackBits() filters = [flt] data = np.random.randint(0, 2, size=100, dtype=bool) for compressor in compressors: - a = array(data, chunks=5, compressor=compressor, filters=filters) # check round-trip @@ -170,14 +160,12 @@ def test_array_with_packbits_filter(): def test_array_with_categorize_filter(): - # setup data = np.random.choice(["foo", "bar", "baz"], size=100) flt = Categorize(dtype=data.dtype, labels=["foo", "bar", "baz"]) filters = [flt] for compressor in compressors: - a = array(data, chunks=5, compressor=compressor, filters=filters) # check round-trip diff --git a/zarr/tests/test_hierarchy.py b/zarr/tests/test_hierarchy.py index cbf59c55c3..6c08d7b88a 100644 --- a/zarr/tests/test_hierarchy.py +++ b/zarr/tests/test_hierarchy.py @@ -1085,7 +1085,6 @@ def test_paths(self): g1.store.close() def test_pickle(self): - # setup group g = self.create_group() d = g.create_dataset("foo/bar", shape=100, chunks=10) @@ -1113,7 +1112,6 @@ def test_pickle(self): g2.store.close() def test_context_manager(self): - with self.create_group() as g: d = g.create_dataset("foo/bar", shape=100, chunks=10) d[:] = np.arange(100) @@ -1375,7 +1373,6 @@ def create_store(): return store, None def test_context_manager(self): - with self.create_group() as g: store = g.store d = g.create_dataset("foo/bar", shape=100, chunks=10) diff --git a/zarr/tests/test_indexing.py b/zarr/tests/test_indexing.py index 8a34c1e715..f10360e8b7 100644 --- a/zarr/tests/test_indexing.py +++ b/zarr/tests/test_indexing.py @@ -17,7 +17,6 @@ def test_normalize_integer_selection(): - assert 1 == normalize_integer_selection(1, 100) assert 99 == normalize_integer_selection(-1, 100) with pytest.raises(IndexError): @@ -29,7 +28,6 @@ def test_normalize_integer_selection(): def test_replace_ellipsis(): - # 1D, single item assert (0,) == replace_ellipsis(0, (100,)) @@ -68,7 +66,6 @@ def test_replace_ellipsis(): def test_get_basic_selection_0d(): - # setup a = np.array(42) z = zarr.create(shape=a.shape, dtype=a.dtype, fill_value=None) @@ -191,7 +188,6 @@ def _test_get_basic_selection(a, z, selection): # noinspection PyStatementEffect def test_get_basic_selection_1d(): - # setup a = np.arange(1050, dtype=int) z = zarr.create(shape=a.shape, chunks=100, dtype=a.dtype) @@ -264,7 +260,6 @@ def test_get_basic_selection_1d(): # noinspection PyStatementEffect def test_get_basic_selection_2d(): - # setup a = np.arange(10000, dtype=int).reshape(1000, 10) z = zarr.create(shape=a.shape, chunks=(300, 3), dtype=a.dtype) @@ -423,7 +418,6 @@ def test_fancy_indexing_doesnt_mix_with_implicit_slicing(): def test_set_basic_selection_0d(): - # setup v = np.array(42) a = np.zeros_like(v) @@ -479,7 +473,6 @@ def _test_get_orthogonal_selection(a, z, selection): # noinspection PyStatementEffect def test_get_orthogonal_selection_1d_bool(): - # setup a = np.arange(1050, dtype=int) z = zarr.create(shape=a.shape, chunks=100, dtype=a.dtype) @@ -502,7 +495,6 @@ def test_get_orthogonal_selection_1d_bool(): # noinspection PyStatementEffect def test_get_orthogonal_selection_1d_int(): - # setup a = np.arange(1050, dtype=int) z = zarr.create(shape=a.shape, chunks=100, dtype=a.dtype) @@ -561,7 +553,6 @@ def _test_get_orthogonal_selection_2d(a, z, ix0, ix1): # noinspection PyStatementEffect def test_get_orthogonal_selection_2d(): - # setup a = np.arange(10000, dtype=int).reshape(1000, 10) z = zarr.create(shape=a.shape, chunks=(300, 3), dtype=a.dtype) @@ -570,7 +561,6 @@ def test_get_orthogonal_selection_2d(): np.random.seed(42) # test with different degrees of sparseness for p in 0.5, 0.1, 0.01: - # boolean arrays ix0 = np.random.binomial(1, p, size=a.shape[0]).astype(bool) ix1 = np.random.binomial(1, 0.5, size=a.shape[1]).astype(bool) @@ -641,7 +631,6 @@ def _test_get_orthogonal_selection_3d(a, z, ix0, ix1, ix2): def test_get_orthogonal_selection_3d(): - # setup a = np.arange(100000, dtype=int).reshape(200, 50, 10) z = zarr.create(shape=a.shape, chunks=(60, 20, 3), dtype=a.dtype) @@ -650,7 +639,6 @@ def test_get_orthogonal_selection_3d(): np.random.seed(42) # test with different degrees of sparseness for p in 0.5, 0.1, 0.01: - # boolean arrays ix0 = np.random.binomial(1, p, size=a.shape[0]).astype(bool) ix1 = np.random.binomial(1, 0.5, size=a.shape[1]).astype(bool) @@ -673,7 +661,6 @@ def test_get_orthogonal_selection_3d(): def test_orthogonal_indexing_edge_cases(): - a = np.arange(6).reshape(1, 2, 3) z = zarr.create(shape=a.shape, chunks=(1, 2, 3), dtype=a.dtype) z[:] = a @@ -706,7 +693,6 @@ def _test_set_orthogonal_selection(v, a, z, selection): def test_set_orthogonal_selection_1d(): - # setup v = np.arange(1050, dtype=int) a = np.empty(v.shape, dtype=int) @@ -715,7 +701,6 @@ def test_set_orthogonal_selection_1d(): # test with different degrees of sparseness np.random.seed(42) for p in 0.5, 0.1, 0.01: - # boolean arrays ix = np.random.binomial(1, p, size=a.shape[0]).astype(bool) _test_set_orthogonal_selection(v, a, z, ix) @@ -734,7 +719,6 @@ def test_set_orthogonal_selection_1d(): def _test_set_orthogonal_selection_2d(v, a, z, ix0, ix1): - selections = [ # index both axes with array (ix0, ix1), @@ -749,7 +733,6 @@ def _test_set_orthogonal_selection_2d(v, a, z, ix0, ix1): def test_set_orthogonal_selection_2d(): - # setup v = np.arange(10000, dtype=int).reshape(1000, 10) a = np.empty_like(v) @@ -758,7 +741,6 @@ def test_set_orthogonal_selection_2d(): np.random.seed(42) # test with different degrees of sparseness for p in 0.5, 0.1, 0.01: - # boolean arrays ix0 = np.random.binomial(1, p, size=a.shape[0]).astype(bool) ix1 = np.random.binomial(1, 0.5, size=a.shape[1]).astype(bool) @@ -780,7 +762,6 @@ def test_set_orthogonal_selection_2d(): def _test_set_orthogonal_selection_3d(v, a, z, ix0, ix1, ix2): - selections = ( # single value (84, 42, 4), @@ -807,7 +788,6 @@ def _test_set_orthogonal_selection_3d(v, a, z, ix0, ix1, ix2): def test_set_orthogonal_selection_3d(): - # setup v = np.arange(100000, dtype=int).reshape(200, 50, 10) a = np.empty_like(v) @@ -816,7 +796,6 @@ def test_set_orthogonal_selection_3d(): np.random.seed(42) # test with different degrees of sparseness for p in 0.5, 0.1, 0.01: - # boolean arrays ix0 = np.random.binomial(1, p, size=a.shape[0]).astype(bool) ix1 = np.random.binomial(1, 0.5, size=a.shape[1]).astype(bool) @@ -888,7 +867,6 @@ def _test_get_coordinate_selection(a, z, selection): # noinspection PyStatementEffect def test_get_coordinate_selection_1d(): - # setup a = np.arange(1050, dtype=int) z = zarr.create(shape=a.shape, chunks=100, dtype=a.dtype) @@ -932,7 +910,6 @@ def test_get_coordinate_selection_1d(): def test_get_coordinate_selection_2d(): - # setup a = np.arange(10000, dtype=int).reshape(1000, 10) z = zarr.create(shape=a.shape, chunks=(300, 3), dtype=a.dtype) @@ -1027,7 +1004,6 @@ def test_set_coordinate_selection_1d(): def test_set_coordinate_selection_2d(): - # setup v = np.arange(10000, dtype=int).reshape(1000, 10) a = np.empty_like(v) @@ -1258,7 +1234,6 @@ def _test_get_mask_selection(a, z, selection): # noinspection PyStatementEffect def test_get_mask_selection_1d(): - # setup a = np.arange(1050, dtype=int) z = zarr.create(shape=a.shape, chunks=100, dtype=a.dtype) @@ -1285,7 +1260,6 @@ def test_get_mask_selection_1d(): # noinspection PyStatementEffect def test_get_mask_selection_2d(): - # setup a = np.arange(10000, dtype=int).reshape(1000, 10) z = zarr.create(shape=a.shape, chunks=(300, 3), dtype=a.dtype) @@ -1318,7 +1292,6 @@ def _test_set_mask_selection(v, a, z, selection): def test_set_mask_selection_1d(): - # setup v = np.arange(1050, dtype=int) a = np.empty_like(v) @@ -1338,7 +1311,6 @@ def test_set_mask_selection_1d(): def test_set_mask_selection_2d(): - # setup v = np.arange(10000, dtype=int).reshape(1000, 10) a = np.empty_like(v) @@ -1352,7 +1324,6 @@ def test_set_mask_selection_2d(): def test_get_selection_out(): - # basic selections a = np.arange(1050) z = zarr.create(shape=1050, chunks=100, dtype=a.dtype) @@ -1426,7 +1397,6 @@ def test_get_selection_out(): def test_get_selections_with_fields(): - a = [("aaa", 1, 4.2), ("bbb", 2, 8.4), ("ccc", 3, 12.6)] a = np.array(a, dtype=[("foo", "S3"), ("bar", "i4"), ("baz", "f8")]) z = zarr.create(shape=a.shape, chunks=2, dtype=a.dtype, fill_value=None) @@ -1444,7 +1414,6 @@ def test_get_selections_with_fields(): ] for fields in fields_fixture: - # total selection expect = a[fields] actual = z.get_basic_selection(Ellipsis, fields=fields) @@ -1534,7 +1503,6 @@ def test_get_selections_with_fields(): def test_set_selections_with_fields(): - v = [("aaa", 1, 4.2), ("bbb", 2, 8.4), ("ccc", 3, 12.6)] v = np.array(v, dtype=[("foo", "S3"), ("bar", "i4"), ("baz", "f8")]) a = np.empty_like(v) @@ -1553,7 +1521,6 @@ def test_set_selections_with_fields(): ] for fields in fields_fixture: - # currently multi-field assignment is not supported in numpy, so we won't support # it either if isinstance(fields, list) and len(fields) > 1: @@ -1567,7 +1534,6 @@ def test_set_selections_with_fields(): z.set_mask_selection([True, False, True], v, fields=fields) else: - if isinstance(fields, list) and len(fields) == 1: # work around numpy does not support multi-field assignment even if there # is only one field @@ -1752,7 +1718,6 @@ def test_accessed_chunks(shape, chunks, ops): z = zarr.create(shape=shape, chunks=chunks, store=store) for ii, (optype, slices) in enumerate(ops): - # Resolve the slices into the accessed chunks for each dimension chunks_per_dim = [] for N, C, sl in zip(shape, chunks, slices): diff --git a/zarr/tests/test_info.py b/zarr/tests/test_info.py index 7fb6feb11b..96eae999f4 100644 --- a/zarr/tests/test_info.py +++ b/zarr/tests/test_info.py @@ -7,7 +7,6 @@ @pytest.mark.parametrize("array_size", [10, 15000]) def test_info(array_size): - # setup g = zarr.group(store=dict(), chunk_store=dict(), synchronizer=zarr.ThreadSynchronizer()) g.create_group("foo") diff --git a/zarr/tests/test_meta.py b/zarr/tests/test_meta.py index db50560c8e..3e1e0f9d63 100644 --- a/zarr/tests/test_meta.py +++ b/zarr/tests/test_meta.py @@ -34,7 +34,6 @@ def assert_json_equal(expect, actual): def test_encode_decode_array_1(): - meta = dict( shape=(100,), chunks=(10,), @@ -76,7 +75,6 @@ def test_encode_decode_array_1(): def test_encode_decode_array_2(): - # some variations df = Delta(astype=" Tupl def normalize_dtype(dtype: Union[str, np.dtype], object_codec) -> Tuple[np.dtype, Any]: - # convenience API for object arrays if inspect.isclass(dtype): dtype = dtype.__name__ # type: ignore @@ -245,7 +244,6 @@ def is_total_slice(item, shape: Tuple[int]) -> bool: def normalize_resize_args(old_shape, *args): - # normalize new shape argument if len(args) == 1: new_shape = args[0] @@ -294,7 +292,6 @@ def normalize_dimension_separator(sep: Optional[str]) -> Optional[str]: def normalize_fill_value(fill_value, dtype: np.dtype): - if fill_value is None or dtype.hasobject: # no fill value pass @@ -332,7 +329,6 @@ def normalize_fill_value(fill_value, dtype: np.dtype): def normalize_storage_path(path: Union[str, bytes, None]) -> str: - # handle bytes if isinstance(path, bytes): path = str(path, "ascii") @@ -342,7 +338,6 @@ def normalize_storage_path(path: Union[str, bytes, None]) -> str: path = str(path) if path: - # convert backslash to forward slash path = path.replace("\\", "/") @@ -506,7 +501,6 @@ def tree_widget(group, expand, level): class TreeViewer: def __init__(self, group, expand=False, level=None): - self.group = group self.expand = expand self.level = level From 54e31e9814a41cd7fd81255695971ce5e700ee3e Mon Sep 17 00:00:00 2001 From: Dimitri Papadopoulos Orfanos <3234522+DimitriPapadopoulos@users.noreply.github.com> Date: Thu, 7 Dec 2023 22:29:28 +0100 Subject: [PATCH 005/130] Use list comprehension where applicable (#1555) Even if this is only a test, list comprehensions are faster than repeatedly call append(). Also use tuple instead of list when possible. Co-authored-by: Davis Bennett --- zarr/tests/test_indexing.py | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/zarr/tests/test_indexing.py b/zarr/tests/test_indexing.py index f10360e8b7..af046e9d28 100644 --- a/zarr/tests/test_indexing.py +++ b/zarr/tests/test_indexing.py @@ -1719,17 +1719,15 @@ def test_accessed_chunks(shape, chunks, ops): for ii, (optype, slices) in enumerate(ops): # Resolve the slices into the accessed chunks for each dimension - chunks_per_dim = [] - for N, C, sl in zip(shape, chunks, slices): - chunk_ind = np.arange(N, dtype=int)[sl] // C - chunks_per_dim.append(np.unique(chunk_ind)) + chunks_per_dim = [ + np.unique(np.arange(N, dtype=int)[sl] // C) for N, C, sl in zip(shape, chunks, slices) + ] # Combine and generate the cartesian product to determine the chunks keys that # will be accessed - chunks_accessed = [] - for comb in itertools.product(*chunks_per_dim): - chunks_accessed.append(".".join([str(ci) for ci in comb])) - + chunks_accessed = ( + ".".join([str(ci) for ci in comb]) for comb in itertools.product(*chunks_per_dim) + ) counts_before = store.counter.copy() # Perform the operation From 7d2c9bf5ce4c998d95630d9a1202e27e58926838 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 7 Dec 2023 21:35:23 +0000 Subject: [PATCH 006/130] Bump numcodecs from 0.11.0 to 0.12.1 (#1580) Bumps [numcodecs](https://github.com/zarr-developers/numcodecs) from 0.11.0 to 0.12.1. - [Release notes](https://github.com/zarr-developers/numcodecs/releases) - [Changelog](https://github.com/zarr-developers/numcodecs/blob/main/docs/release.rst) - [Commits](https://github.com/zarr-developers/numcodecs/compare/v0.11.0...v0.12.1) --- updated-dependencies: - dependency-name: numcodecs dependency-type: direct:development update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Joe Hamman --- requirements_dev_minimal.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements_dev_minimal.txt b/requirements_dev_minimal.txt index e2be6eb825..afea816d87 100644 --- a/requirements_dev_minimal.txt +++ b/requirements_dev_minimal.txt @@ -1,7 +1,7 @@ # library requirements asciitree==0.3.3 fasteners==0.19 -numcodecs==0.11.0 +numcodecs==0.12.1 msgpack-python==0.5.6 setuptools-scm==8.0.4 # test requirements From 10dee6ba0c0ce6ab29333e7a50f0afa4f6de06ca Mon Sep 17 00:00:00 2001 From: Dimitri Papadopoulos Orfanos <3234522+DimitriPapadopoulos@users.noreply.github.com> Date: Thu, 7 Dec 2023 22:40:21 +0100 Subject: [PATCH 007/130] Use format specification mini-language to format string (#1558) Co-authored-by: Joe Hamman --- zarr/storage.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zarr/storage.py b/zarr/storage.py index 585417f59c..5ba8071395 100644 --- a/zarr/storage.py +++ b/zarr/storage.py @@ -784,7 +784,7 @@ def __len__(self): return len(self._mutable_mapping) def __repr__(self): - return f"<{self.__class__.__name__}: \n{repr(self._mutable_mapping)}\n at {hex(id(self))}>" + return f"<{self.__class__.__name__}: \n{self._mutable_mapping!r}\n at {id(self):#x}>" def __eq__(self, other): if isinstance(other, KVStore): From 40a6e817b17e1fe600b188478ba38fb6978a5273 Mon Sep 17 00:00:00 2001 From: Dimitri Papadopoulos Orfanos <3234522+DimitriPapadopoulos@users.noreply.github.com> Date: Thu, 7 Dec 2023 22:45:50 +0100 Subject: [PATCH 008/130] Single startswith() call instead of multiple ones (#1556) It's faster and probably more readable. Co-authored-by: Davis Bennett Co-authored-by: Joe Hamman --- zarr/_storage/store.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/zarr/_storage/store.py b/zarr/_storage/store.py index 80e4ad8f75..667ca38147 100644 --- a/zarr/_storage/store.py +++ b/zarr/_storage/store.py @@ -221,9 +221,8 @@ def _validate_key(self, key: str): ) if ( - not key.startswith("data/") - and (not key.startswith("meta/")) - and (not key == "zarr.json") + not key.startswith(("data/", "meta/")) + and key != "zarr.json" # TODO: Possibly allow key == ".zmetadata" too if we write a # consolidated metadata spec corresponding to this? ): From 5954ff95803c1343d022f6181ed397c7095f4a0e Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 7 Dec 2023 21:46:14 +0000 Subject: [PATCH 009/130] Bump pymongo from 4.5.0 to 4.6.1 (#1585) Bumps [pymongo](https://github.com/mongodb/mongo-python-driver) from 4.5.0 to 4.6.1. - [Release notes](https://github.com/mongodb/mongo-python-driver/releases) - [Changelog](https://github.com/mongodb/mongo-python-driver/blob/master/doc/changelog.rst) - [Commits](https://github.com/mongodb/mongo-python-driver/compare/4.5.0...4.6.1) --- updated-dependencies: - dependency-name: pymongo dependency-type: direct:development update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Joe Hamman --- requirements_dev_optional.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements_dev_optional.txt b/requirements_dev_optional.txt index f3ea80a546..5a3340a282 100644 --- a/requirements_dev_optional.txt +++ b/requirements_dev_optional.txt @@ -11,7 +11,7 @@ azure-storage-blob==12.16.0 # pyup: ignore redis==5.0.1 types-redis types-setuptools -pymongo==4.5.0 +pymongo==4.6.1 # optional test requirements coverage pytest-cov==4.1.0 From 6ad7b0e2ddabdcc5087e23f003edf123d21e9a25 Mon Sep 17 00:00:00 2001 From: Dimitri Papadopoulos Orfanos <3234522+DimitriPapadopoulos@users.noreply.github.com> Date: Fri, 8 Dec 2023 00:07:10 +0100 Subject: [PATCH 010/130] Move codespell options around (#1196) Starting with codespell 2.2.2, options can be specified in `pyrpoject.toml` in addition to `setup.cfg`: https://github.com/codespell-project/codespell#using-a-config-file Specifying options in a config file instead of command line options in `.pre-commit-config.yaml` ensures codespell uses the same options when run as pre-commit hook or from the command line in the repository root directory. --- .pre-commit-config.yaml | 1 - pyproject.toml | 5 +++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index e985d24000..029dcda58f 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -21,7 +21,6 @@ repos: rev: v2.2.5 hooks: - id: codespell - args: ["-L", "ba,ihs,kake,nd,noe,nwo,te,fo,zar", "-S", "fixture"] - repo: https://github.com/pre-commit/pre-commit-hooks rev: v4.4.0 hooks: diff --git a/pyproject.toml b/pyproject.toml index 22ea19f28f..36a0d896ea 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -137,3 +137,8 @@ filterwarnings = [ "ignore:PY_SSIZE_T_CLEAN will be required.*:DeprecationWarning", "ignore:The loop argument is deprecated since Python 3.8.*:DeprecationWarning", ] + + +[tool.codespell] +ignore-words-list = "ba,ihs,kake,nd,noe,nwo,te,fo,zar" +skip = 'fixture,.git' From cf32382b9a228eaaafe30ab82d05b9303824a783 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 8 Dec 2023 15:19:14 +0100 Subject: [PATCH 011/130] Bump fsspec from 2023.10.0 to 2023.12.1 (#1600) * Bump fsspec from 2023.10.0 to 2023.12.1 Bumps [fsspec](https://github.com/fsspec/filesystem_spec) from 2023.10.0 to 2023.12.1. - [Commits](https://github.com/fsspec/filesystem_spec/compare/2023.10.0...2023.12.1) --- updated-dependencies: - dependency-name: fsspec dependency-type: direct:development update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] * Update s3fs as well * Fix s3fs --------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Josh Moore --- requirements_dev_optional.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements_dev_optional.txt b/requirements_dev_optional.txt index 5a3340a282..13385a243a 100644 --- a/requirements_dev_optional.txt +++ b/requirements_dev_optional.txt @@ -18,6 +18,6 @@ pytest-cov==4.1.0 pytest-doctestplus==1.0.0 pytest-timeout==2.2.0 h5py==3.10.0 -fsspec==2023.10.0 -s3fs==2023.10.0 +fsspec==2023.12.1 +s3fs==2023.12.1 moto[server]>=4.0.8 From 4d79cfc84f7f3914a04d9468666685520cc21276 Mon Sep 17 00:00:00 2001 From: David Stansby Date: Fri, 8 Dec 2023 16:41:51 +0000 Subject: [PATCH 012/130] Add type hints to zarr.create (#1536) * Add type hints to zarr.create * Use protocol for MetaArray * Use protocol for Synchronizer * Fix Path typing * Add release note * Fix dim separator typing * Ignore ... in coverage reporting * Fix chunk typing --------- Co-authored-by: Davis Bennett --- docs/release.rst | 6 ++++++ pyproject.toml | 1 + zarr/_storage/store.py | 3 ++- zarr/creation.py | 46 +++++++++++++++++++++++------------------ zarr/storage.py | 2 +- zarr/sync.py | 12 +++++++++-- zarr/tests/test_core.py | 8 ++++--- zarr/types.py | 13 ++++++++++++ zarr/util.py | 5 +++-- 9 files changed, 67 insertions(+), 29 deletions(-) create mode 100644 zarr/types.py diff --git a/docs/release.rst b/docs/release.rst index 842c36e290..c18e0b8c20 100644 --- a/docs/release.rst +++ b/docs/release.rst @@ -18,6 +18,12 @@ Release notes Unreleased ---------- +Enhancements +~~~~~~~~~~~~ + +* Added type hints to ``zarr.creation.create()``. + By :user:`David Stansby ` :issue:`1536`. + Docs ~~~~ diff --git a/pyproject.toml b/pyproject.toml index 36a0d896ea..4b7fef6003 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -65,6 +65,7 @@ Homepage = "https://github.com/zarr-developers/zarr-python" exclude_lines = [ "pragma: no cover", "pragma: ${PY_MAJOR_VERSION} no cover", + '.*\.\.\.' # Ignore "..." lines ] [tool.coverage.run] diff --git a/zarr/_storage/store.py b/zarr/_storage/store.py index 667ca38147..09f0b68602 100644 --- a/zarr/_storage/store.py +++ b/zarr/_storage/store.py @@ -9,6 +9,7 @@ from zarr.meta import Metadata2, Metadata3 from zarr.util import normalize_storage_path from zarr.context import Context +from zarr.types import ZARR_VERSION # v2 store keys array_meta_key = ".zarray" @@ -19,7 +20,7 @@ meta_root = "meta/root/" data_root = "data/root/" -DEFAULT_ZARR_VERSION = 2 +DEFAULT_ZARR_VERSION: ZARR_VERSION = 2 v3_api_available = os.environ.get("ZARR_V3_EXPERIMENTAL_API", "0").lower() not in ["0", "false"] diff --git a/zarr/creation.py b/zarr/creation.py index 6227f90b7b..d4f570895a 100644 --- a/zarr/creation.py +++ b/zarr/creation.py @@ -1,7 +1,10 @@ -from typing import Optional +from collections.abc import MutableMapping +from typing import Optional, Tuple, Union, Sequence from warnings import warn import numpy as np +import numpy.typing as npt +from numcodecs.abc import Codec from numcodecs.registry import codec_registry from zarr._storage.store import DEFAULT_ZARR_VERSION @@ -19,32 +22,35 @@ normalize_storage_path, normalize_store_arg, ) +from zarr._storage.store import StorageTransformer +from zarr.sync import Synchronizer +from zarr.types import ZARR_VERSION, DIMENSION_SEPARATOR, MEMORY_ORDER, MetaArray, PathLike from zarr.util import normalize_dimension_separator def create( - shape, - chunks=True, - dtype=None, + shape: Union[int, Tuple[int, ...]], + chunks: Union[int, Tuple[int, ...], bool] = True, + dtype: Optional[npt.DTypeLike] = None, compressor="default", fill_value: Optional[int] = 0, - order="C", - store=None, - synchronizer=None, - overwrite=False, - path=None, - chunk_store=None, - filters=None, - cache_metadata=True, - cache_attrs=True, - read_only=False, - object_codec=None, - dimension_separator=None, - write_empty_chunks=True, + order: MEMORY_ORDER = "C", + store: Optional[Union[str, MutableMapping]] = None, + synchronizer: Optional[Synchronizer] = None, + overwrite: bool = False, + path: Optional[PathLike] = None, + chunk_store: Optional[MutableMapping] = None, + filters: Optional[Sequence[Codec]] = None, + cache_metadata: bool = True, + cache_attrs: bool = True, + read_only: bool = False, + object_codec: Optional[Codec] = None, + dimension_separator: Optional[DIMENSION_SEPARATOR] = None, + write_empty_chunks: bool = True, *, - zarr_version=None, - meta_array=None, - storage_transformers=(), + zarr_version: Optional[ZARR_VERSION] = None, + meta_array: Optional[MetaArray] = None, + storage_transformers: Sequence[StorageTransformer] = (), **kwargs, ): """Create an array. diff --git a/zarr/storage.py b/zarr/storage.py index 5ba8071395..1c3b39862a 100644 --- a/zarr/storage.py +++ b/zarr/storage.py @@ -40,6 +40,7 @@ from numcodecs.compat import ensure_bytes, ensure_text, ensure_contiguous_ndarray_like from numcodecs.registry import codec_registry from zarr.context import Context +from zarr.types import PathLike as Path from zarr.errors import ( MetadataError, @@ -105,7 +106,6 @@ default_compressor = Zlib() -Path = Union[str, bytes, None] # allow MutableMapping for backwards compatibility StoreLike = Union[BaseStore, MutableMapping] diff --git a/zarr/sync.py b/zarr/sync.py index 49684a51ee..2e843f6557 100644 --- a/zarr/sync.py +++ b/zarr/sync.py @@ -1,11 +1,19 @@ import os from collections import defaultdict from threading import Lock +from typing import Protocol import fasteners -class ThreadSynchronizer: +class Synchronizer(Protocol): + """Base class for synchronizers.""" + + def __getitem__(self, item): + ... + + +class ThreadSynchronizer(Synchronizer): """Provides synchronization using thread locks.""" def __init__(self): @@ -24,7 +32,7 @@ def __setstate__(self, *args): self.__init__() -class ProcessSynchronizer: +class ProcessSynchronizer(Synchronizer): """Provides synchronization using file locks via the `fasteners `_ package. diff --git a/zarr/tests/test_core.py b/zarr/tests/test_core.py index f3ca73dea8..a3fde4050d 100644 --- a/zarr/tests/test_core.py +++ b/zarr/tests/test_core.py @@ -3,7 +3,7 @@ import sys import pickle import shutil -from typing import Any, Literal, Optional, Tuple, Union +from typing import Any, Literal, Optional, Tuple, Union, Sequence import unittest from itertools import zip_longest from tempfile import mkdtemp @@ -26,6 +26,7 @@ VLenUTF8, Zlib, ) +from numcodecs.abc import Codec from numcodecs.compat import ensure_bytes, ensure_ndarray from numcodecs.tests.common import greetings from numpy.testing import assert_array_almost_equal, assert_array_equal @@ -73,6 +74,7 @@ from zarr.tests.test_storage_v3 import DummyStorageTransfomer from zarr.util import buffer_size from zarr.tests.util import abs_container, skip_test_env_var, have_fsspec, mktemp +from zarr.types import DIMENSION_SEPARATOR # noinspection PyMethodMayBeStatic @@ -82,8 +84,8 @@ class TestArray: root = "" path = "" compressor = Zlib(level=1) - filters = None - dimension_separator: Literal["/", ".", None] = None + filters: Optional[Sequence[Codec]] = None + dimension_separator: Optional[DIMENSION_SEPARATOR] = None cache_metadata = True cache_attrs = True partial_decompress: bool = False diff --git a/zarr/types.py b/zarr/types.py new file mode 100644 index 0000000000..1de270f25c --- /dev/null +++ b/zarr/types.py @@ -0,0 +1,13 @@ +from typing import Literal, Protocol, Union + +ZARR_VERSION = Literal[2, 3] +DIMENSION_SEPARATOR = Literal[".", "/"] +MEMORY_ORDER = Literal["C", "F"] + + +PathLike = Union[str, bytes, None] + + +class MetaArray(Protocol): + def __array_function__(self, func, types, args, kwargs): + ... diff --git a/zarr/util.py b/zarr/util.py index df1cd9d409..f97094b93a 100644 --- a/zarr/util.py +++ b/zarr/util.py @@ -31,6 +31,7 @@ from numcodecs.ndarray_like import NDArrayLike from numcodecs.registry import codec_registry from numcodecs.blosc import cbuffer_sizes, cbuffer_metainfo +from zarr.types import DIMENSION_SEPARATOR KeyType = TypeVar("KeyType") ValueType = TypeVar("ValueType") @@ -284,9 +285,9 @@ def normalize_order(order: str) -> str: return order -def normalize_dimension_separator(sep: Optional[str]) -> Optional[str]: +def normalize_dimension_separator(sep: Optional[str]) -> Optional[DIMENSION_SEPARATOR]: if sep in (".", "/", None): - return sep + return cast(Optional[DIMENSION_SEPARATOR], sep) else: raise ValueError("dimension_separator must be either '.' or '/', found: %r" % sep) From 12abd4e434e816e9b8f19b1ceb89438fa4269737 Mon Sep 17 00:00:00 2001 From: David Stansby Date: Mon, 18 Dec 2023 11:57:44 +0000 Subject: [PATCH 013/130] Remove unused mypy ignore comments (#1602) Co-authored-by: Davis Bennett --- pyproject.toml | 5 +++-- zarr/_storage/store.py | 4 ++-- zarr/_storage/v3_storage_transformers.py | 2 +- zarr/meta.py | 4 ++-- zarr/storage.py | 12 ++++++------ zarr/util.py | 2 +- 6 files changed, 15 insertions(+), 14 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 4b7fef6003..33e8573830 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -120,9 +120,10 @@ exclude = ''' ''' [tool.mypy] -python_version = "3.8" ignore_missing_imports = true -follow_imports = "silent" +warn_unused_configs = true +warn_redundant_casts = true +warn_unused_ignores = true [tool.pytest.ini_options] doctest_optionflags = [ diff --git a/zarr/_storage/store.py b/zarr/_storage/store.py index 09f0b68602..36b596769a 100644 --- a/zarr/_storage/store.py +++ b/zarr/_storage/store.py @@ -642,10 +642,10 @@ def _rmdir_from_keys_v3(store: StoreV3, path: str = "") -> None: sfx = _get_metadata_suffix(store) array_meta_file = meta_dir + ".array" + sfx if array_meta_file in store: - store.erase(array_meta_file) # type: ignore + store.erase(array_meta_file) group_meta_file = meta_dir + ".group" + sfx if group_meta_file in store: - store.erase(group_meta_file) # type: ignore + store.erase(group_meta_file) def _listdir_from_keys(store: BaseStore, path: Optional[str] = None) -> List[str]: diff --git a/zarr/_storage/v3_storage_transformers.py b/zarr/_storage/v3_storage_transformers.py index ff31a7281c..3afc3823a3 100644 --- a/zarr/_storage/v3_storage_transformers.py +++ b/zarr/_storage/v3_storage_transformers.py @@ -351,7 +351,7 @@ def erase_prefix(self, prefix): def rmdir(self, path=None): path = normalize_storage_path(path) - _rmdir_from_keys_v3(self, path) # type: ignore + _rmdir_from_keys_v3(self, path) def __contains__(self, key): if self._is_data_key(key): diff --git a/zarr/meta.py b/zarr/meta.py index f23889f3ea..d9797e4754 100644 --- a/zarr/meta.py +++ b/zarr/meta.py @@ -234,8 +234,8 @@ def decode_fill_value(cls, v: Any, dtype: np.dtype, object_codec: Any = None) -> return np.array(v, dtype=dtype)[()] elif dtype.kind in "c": v = ( - cls.decode_fill_value(v[0], dtype.type().real.dtype), # type: ignore - cls.decode_fill_value(v[1], dtype.type().imag.dtype), # type: ignore + cls.decode_fill_value(v[0], dtype.type().real.dtype), + cls.decode_fill_value(v[1], dtype.type().imag.dtype), ) v = v[0] + 1j * v[1] return np.array(v, dtype=dtype)[()] diff --git a/zarr/storage.py b/zarr/storage.py index 1c3b39862a..aa27e98e6f 100644 --- a/zarr/storage.py +++ b/zarr/storage.py @@ -206,7 +206,7 @@ def rmdir(store: StoreLike, path: Path = None): store_version = getattr(store, "_store_version", 2) if hasattr(store, "rmdir") and store.is_erasable(): # type: ignore # pass through - store.rmdir(path) # type: ignore + store.rmdir(path) else: # slow version, delete one key at a time if store_version == 2: @@ -236,7 +236,7 @@ def listdir(store: BaseStore, path: Path = None): path = normalize_storage_path(path) if hasattr(store, "listdir"): # pass through - return store.listdir(path) # type: ignore + return store.listdir(path) else: # slow version, iterate through all keys warnings.warn( @@ -289,7 +289,7 @@ def getsize(store: BaseStore, path: Path = None) -> int: if hasattr(store, "getsize"): # pass through path = normalize_storage_path(path) - return store.getsize(path) # type: ignore + return store.getsize(path) elif isinstance(store, MutableMapping): return _getsize(store, path) else: @@ -627,7 +627,7 @@ def _init_array_metadata( key = _prefix_to_array_key(store, _path_to_prefix(path)) if hasattr(store, "_metadata_class"): - store[key] = store._metadata_class.encode_array_metadata(meta) # type: ignore + store[key] = store._metadata_class.encode_array_metadata(meta) else: store[key] = encode_array_metadata(meta) @@ -730,10 +730,10 @@ def _init_group_metadata( if store_version == 3: meta = {"attributes": {}} # type: ignore else: - meta = {} # type: ignore + meta = {} key = _prefix_to_group_key(store, _path_to_prefix(path)) if hasattr(store, "_metadata_class"): - store[key] = store._metadata_class.encode_group_metadata(meta) # type: ignore + store[key] = store._metadata_class.encode_group_metadata(meta) else: store[key] = encode_group_metadata(meta) diff --git a/zarr/util.py b/zarr/util.py index f97094b93a..54c389db69 100644 --- a/zarr/util.py +++ b/zarr/util.py @@ -183,7 +183,7 @@ def normalize_chunks(chunks: Any, shape: Tuple[int, ...], typesize: int) -> Tupl def normalize_dtype(dtype: Union[str, np.dtype], object_codec) -> Tuple[np.dtype, Any]: # convenience API for object arrays if inspect.isclass(dtype): - dtype = dtype.__name__ # type: ignore + dtype = dtype.__name__ if isinstance(dtype, str): # allow ':' to delimit class from codec arguments tokens = dtype.split(":") From c2f5f0058d25eaa6695334e399b7b3a2d23f7a10 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 19 Dec 2023 20:47:31 -0700 Subject: [PATCH 014/130] Bump actions/setup-python from 4.7.1 to 5.0.0 (#1605) Bumps [actions/setup-python](https://github.com/actions/setup-python) from 4.7.1 to 5.0.0. - [Release notes](https://github.com/actions/setup-python/releases) - [Commits](https://github.com/actions/setup-python/compare/v4.7.1...v5.0.0) --- updated-dependencies: - dependency-name: actions/setup-python dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Joe Hamman --- .github/workflows/releases.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/releases.yml b/.github/workflows/releases.yml index 3bd25bfbf7..8d8512294d 100644 --- a/.github/workflows/releases.yml +++ b/.github/workflows/releases.yml @@ -16,7 +16,7 @@ jobs: submodules: true fetch-depth: 0 - - uses: actions/setup-python@v4.7.1 + - uses: actions/setup-python@v5.0.0 name: Install Python with: python-version: '3.8' From 490e0fe4e59f234cde85b103252acefa34927184 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 20 Dec 2023 22:01:40 +0100 Subject: [PATCH 015/130] Bump github/codeql-action from 2 to 3 (#1609) Bumps [github/codeql-action](https://github.com/github/codeql-action) from 2 to 3. - [Release notes](https://github.com/github/codeql-action/releases) - [Changelog](https://github.com/github/codeql-action/blob/main/CHANGELOG.md) - [Commits](https://github.com/github/codeql-action/compare/v2...v3) --- updated-dependencies: - dependency-name: github/codeql-action dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/codeql-analysis.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 7013f1784f..bb3d433629 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -42,7 +42,7 @@ jobs: # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL - uses: github/codeql-action/init@v2 + uses: github/codeql-action/init@v3 with: languages: ${{ matrix.language }} # If you wish to specify custom queries, you can do so here or in a config file. @@ -56,7 +56,7 @@ jobs: # Autobuild attempts to build any compiled languages (C/C++, C#, or Java). # If this step fails, then you should remove it and run the build manually (see below) - name: Autobuild - uses: github/codeql-action/autobuild@v2 + uses: github/codeql-action/autobuild@v3 # ℹ️ Command-line programs to run using the OS shell. # 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun @@ -69,4 +69,4 @@ jobs: # ./location_of_script_within_repo/buildscript.sh - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@v2 + uses: github/codeql-action/analyze@v3 From b5f79ddfe7821cc9387fc4084bd7672f59215400 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 20 Dec 2023 14:38:04 -0700 Subject: [PATCH 016/130] chore: update pre-commit hooks (#1448) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * chore: update pre-commit hooks updates: - https://github.com/charliermarsh/ruff-pre-commit → https://github.com/astral-sh/ruff-pre-commit - [github.com/astral-sh/ruff-pre-commit: v0.0.224 → v0.1.8](https://github.com/astral-sh/ruff-pre-commit/compare/v0.0.224...v0.1.8) - [github.com/psf/black: 23.10.1 → 23.12.0](https://github.com/psf/black/compare/23.10.1...23.12.0) - [github.com/codespell-project/codespell: v2.2.5 → v2.2.6](https://github.com/codespell-project/codespell/compare/v2.2.5...v2.2.6) - [github.com/pre-commit/pre-commit-hooks: v4.4.0 → v4.5.0](https://github.com/pre-commit/pre-commit-hooks/compare/v4.4.0...v4.5.0) - [github.com/pre-commit/mirrors-mypy: v1.3.0 → v1.7.1](https://github.com/pre-commit/mirrors-mypy/compare/v1.3.0...v1.7.1) * Attempt to fix ruff * Use isinstance --------- Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Co-authored-by: Josh Moore --- .pre-commit-config.yaml | 14 ++++++-------- zarr/core.py | 2 +- 2 files changed, 7 insertions(+), 9 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 029dcda58f..b4e7ab3ccf 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -6,27 +6,25 @@ default_stages: [commit, push] default_language_version: python: python3 repos: - - repo: https://github.com/charliermarsh/ruff-pre-commit + - repo: https://github.com/astral-sh/ruff-pre-commit # Ruff version. - rev: 'v0.0.224' + rev: 'v0.1.8' hooks: - id: ruff - # Respect `exclude` and `extend-exclude` settings. - args: ["--force-exclude"] - repo: https://github.com/psf/black - rev: 23.10.1 + rev: 23.12.0 hooks: - id: black - repo: https://github.com/codespell-project/codespell - rev: v2.2.5 + rev: v2.2.6 hooks: - id: codespell - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.4.0 + rev: v4.5.0 hooks: - id: check-yaml - repo: https://github.com/pre-commit/mirrors-mypy - rev: v1.3.0 + rev: v1.7.1 hooks: - id: mypy files: zarr diff --git a/zarr/core.py b/zarr/core.py index c07a31e95f..d22a9d79c3 100644 --- a/zarr/core.py +++ b/zarr/core.py @@ -2536,7 +2536,7 @@ def hexdigest(self, hashname="sha1"): checksum = binascii.hexlify(self.digest(hashname=hashname)) # This is a bytes object on Python 3 and we want a str. - if type(checksum) is not str: + if not isinstance(checksum, str): checksum = checksum.decode("utf8") return checksum From e09ee149c4525213b07ace9eaf914ca9f552a703 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 26 Dec 2023 10:01:20 -0700 Subject: [PATCH 017/130] chore: update pre-commit hooks (#1618) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit updates: - [github.com/astral-sh/ruff-pre-commit: v0.1.8 → v0.1.9](https://github.com/astral-sh/ruff-pre-commit/compare/v0.1.8...v0.1.9) - [github.com/psf/black: 23.12.0 → 23.12.1](https://github.com/psf/black/compare/23.12.0...23.12.1) - [github.com/pre-commit/mirrors-mypy: v1.7.1 → v1.8.0](https://github.com/pre-commit/mirrors-mypy/compare/v1.7.1...v1.8.0) Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- .pre-commit-config.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index b4e7ab3ccf..80d3439dc7 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -8,11 +8,11 @@ default_language_version: repos: - repo: https://github.com/astral-sh/ruff-pre-commit # Ruff version. - rev: 'v0.1.8' + rev: 'v0.1.9' hooks: - id: ruff - repo: https://github.com/psf/black - rev: 23.12.0 + rev: 23.12.1 hooks: - id: black - repo: https://github.com/codespell-project/codespell @@ -24,7 +24,7 @@ repos: hooks: - id: check-yaml - repo: https://github.com/pre-commit/mirrors-mypy - rev: v1.7.1 + rev: v1.8.0 hooks: - id: mypy files: zarr From cd139895b45a2d7d347c29b703aa2f6775a1e7c9 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 26 Dec 2023 10:24:59 -0700 Subject: [PATCH 018/130] Bump fsspec from 2023.12.1 to 2023.12.2 (#1606) * Bump fsspec from 2023.12.1 to 2023.12.2 Bumps [fsspec](https://github.com/fsspec/filesystem_spec) from 2023.12.1 to 2023.12.2. - [Commits](https://github.com/fsspec/filesystem_spec/compare/2023.12.1...2023.12.2) --- updated-dependencies: - dependency-name: fsspec dependency-type: direct:development update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] * Update requirements_dev_optional.txt --------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Joe Hamman --- requirements_dev_optional.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements_dev_optional.txt b/requirements_dev_optional.txt index 13385a243a..5916083cfc 100644 --- a/requirements_dev_optional.txt +++ b/requirements_dev_optional.txt @@ -18,6 +18,6 @@ pytest-cov==4.1.0 pytest-doctestplus==1.0.0 pytest-timeout==2.2.0 h5py==3.10.0 -fsspec==2023.12.1 -s3fs==2023.12.1 +fsspec==2023.12.2 +s3fs==2023.12.2 moto[server]>=4.0.8 From 5fb420fcfbabd484e663c78e55d04edd4ac9e486 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 27 Dec 2023 11:32:02 +0100 Subject: [PATCH 019/130] Bump pytest-doctestplus from 1.0.0 to 1.1.0 (#1619) Bumps [pytest-doctestplus](https://github.com/scientific-python/pytest-doctestplus) from 1.0.0 to 1.1.0. - [Release notes](https://github.com/scientific-python/pytest-doctestplus/releases) - [Changelog](https://github.com/scientific-python/pytest-doctestplus/blob/main/CHANGES.rst) - [Commits](https://github.com/scientific-python/pytest-doctestplus/compare/v1.0.0...v1.1.0) --- updated-dependencies: - dependency-name: pytest-doctestplus dependency-type: direct:development update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- requirements_dev_optional.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements_dev_optional.txt b/requirements_dev_optional.txt index 5916083cfc..b4de5fd515 100644 --- a/requirements_dev_optional.txt +++ b/requirements_dev_optional.txt @@ -15,7 +15,7 @@ pymongo==4.6.1 # optional test requirements coverage pytest-cov==4.1.0 -pytest-doctestplus==1.0.0 +pytest-doctestplus==1.1.0 pytest-timeout==2.2.0 h5py==3.10.0 fsspec==2023.12.2 From 435a7ca7306fc31dc880ed23631e3af61bf53d66 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 1 Jan 2024 11:26:58 -0700 Subject: [PATCH 020/130] Bump pytest from 7.4.3 to 7.4.4 (#1622) --- requirements_dev_minimal.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements_dev_minimal.txt b/requirements_dev_minimal.txt index afea816d87..94d3fff8a6 100644 --- a/requirements_dev_minimal.txt +++ b/requirements_dev_minimal.txt @@ -5,4 +5,4 @@ numcodecs==0.12.1 msgpack-python==0.5.6 setuptools-scm==8.0.4 # test requirements -pytest==7.4.3 +pytest==7.4.4 From 6961fa9fb87ed73c85f979d84bfe65238933b5ae Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 8 Jan 2024 20:31:31 -0800 Subject: [PATCH 021/130] chore: update pre-commit hooks (#1626) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit updates: - [github.com/astral-sh/ruff-pre-commit: v0.1.9 → v0.1.11](https://github.com/astral-sh/ruff-pre-commit/compare/v0.1.9...v0.1.11) Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 80d3439dc7..340366ef53 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -8,7 +8,7 @@ default_language_version: repos: - repo: https://github.com/astral-sh/ruff-pre-commit # Ruff version. - rev: 'v0.1.9' + rev: 'v0.1.11' hooks: - id: ruff - repo: https://github.com/psf/black From ee518358d888caaabb6157c0498cb231d2ddb7a7 Mon Sep 17 00:00:00 2001 From: Joe Hamman Date: Wed, 10 Jan 2024 06:47:36 -0800 Subject: [PATCH 022/130] Create TEAM.md (#1628) --- TEAM.md | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) create mode 100644 TEAM.md diff --git a/TEAM.md b/TEAM.md new file mode 100644 index 0000000000..a95885ebe5 --- /dev/null +++ b/TEAM.md @@ -0,0 +1,25 @@ +## Active core-developers +- @joshmoore (Josh Moore) +- @jni (Juan Nunez-Iglesias) +- @rabernat (Ryan Abernathey) +- @jhamman (Joe Hamman) +- @d-v-b (Davis Bennett) +- @jakirkham (jakirkham) +- @martindurant (Martin Durant) + +## Emeritus core-developers +- @alimanfoo (Alistair Miles) +- @shoyer (Stephan Hoyer) +- @ryan-williams (Ryan Williams) +- @jrbourbeau (James Bourbeau) +- @mzjp2 (Zain Patel) +- @grlee77 (Gregory Lee) + +## Former core-developers +- @jeromekelleher (Jerome Kelleher) +- @tjcrone (Tim Crone) +- @funkey (Jan Funke) +- @shikharsg +- @Carreau (Matthias Bussonnier) +- @dazzag24 +- @WardF (Ward Fisher) From c7d66b4f8d7e9a4d50e5e01e5484ff8df612cb51 Mon Sep 17 00:00:00 2001 From: Josh Moore Date: Wed, 10 Jan 2024 20:52:42 +0100 Subject: [PATCH 023/130] Drop python 3.8 and numpy 1.20 (#1557) * Drop 3.8 and add 3.12 * Try removing line_profiler * Also bump the minimal numpy to 1.21 * Drop 3.12 again * Revert "Try removing line_profiler" This reverts commit 837854bec99a9d25aece2ead9666f01690d228cc. * Update release.rst --------- Co-authored-by: Joe Hamman Co-authored-by: jakirkham --- .github/ISSUE_TEMPLATE/bug_report.yml | 2 +- .github/workflows/python-package.yml | 8 ++++---- .github/workflows/releases.yml | 2 +- .github/workflows/windows-testing.yml | 2 +- docs/release.rst | 3 +++ environment.yml | 2 +- pyproject.toml | 5 ++--- 7 files changed, 13 insertions(+), 11 deletions(-) diff --git a/.github/ISSUE_TEMPLATE/bug_report.yml b/.github/ISSUE_TEMPLATE/bug_report.yml index ba05f23fcc..ec98af029e 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.yml +++ b/.github/ISSUE_TEMPLATE/bug_report.yml @@ -27,7 +27,7 @@ body: attributes: label: Python Version description: Version of Python interpreter - placeholder: 3.8.5, 3.9, 3.10, etc. + placeholder: 3.9, 3.10, 3.11, etc. validations: required: true - type: input diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index 0c3c49d78d..d74df9ce67 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -15,13 +15,13 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: ['3.8', '3.9', '3.10', '3.11'] - numpy_version: ['>=1.22.0', '==1.20.*'] + python-version: ['3.9', '3.10', '3.11'] + numpy_version: ['>=1.22.0', '==1.21.*'] exclude: - python-version: '3.10' - numpy_version: '==1.20.*' + numpy_version: '==1.21.*' - python-version: '3.11' - numpy_version: '==1.20.*' + numpy_version: '==1.21.*' services: redis: image: redis diff --git a/.github/workflows/releases.yml b/.github/workflows/releases.yml index 8d8512294d..31a7e2770c 100644 --- a/.github/workflows/releases.yml +++ b/.github/workflows/releases.yml @@ -19,7 +19,7 @@ jobs: - uses: actions/setup-python@v5.0.0 name: Install Python with: - python-version: '3.8' + python-version: '3.9' - name: Install PyBuild run: | diff --git a/.github/workflows/windows-testing.yml b/.github/workflows/windows-testing.yml index eeee5b704d..5c3252c0ba 100644 --- a/.github/workflows/windows-testing.yml +++ b/.github/workflows/windows-testing.yml @@ -16,7 +16,7 @@ jobs: strategy: fail-fast: True matrix: - python-version: ['3.8', '3.9', '3.10', '3.11'] + python-version: ['3.9', '3.10', '3.11'] steps: - uses: actions/checkout@v4 with: diff --git a/docs/release.rst b/docs/release.rst index c18e0b8c20..a3e0831ba4 100644 --- a/docs/release.rst +++ b/docs/release.rst @@ -49,6 +49,9 @@ Docs Maintenance ~~~~~~~~~~~ +* Drop Python 3.8 and NumPy 1.20 + By :user:`Josh Moore `; :issue:`1557`. + * Cache result of ``FSStore._fsspec_installed()``. By :user:`Janick Martinez Esturo ` :issue:`1581`. diff --git a/environment.yml b/environment.yml index dc99507427..ff2f9eedef 100644 --- a/environment.yml +++ b/environment.yml @@ -4,7 +4,7 @@ channels: dependencies: - wheel - numcodecs >= 0.6.4 - - numpy >= 1.20 + - numpy >= 1.21 - pip - pip: - asciitree diff --git a/pyproject.toml b/pyproject.toml index 33e8573830..a85e49e82c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -10,10 +10,10 @@ readme = { file = "README.md", content-type = "text/markdown" } maintainers = [ { name = "Alistair Miles", email = "alimanfoo@googlemail.com" } ] -requires-python = ">=3.8" +requires-python = ">=3.9" dependencies = [ 'asciitree', - 'numpy>=1.20,!=1.21.0', + 'numpy>=1.21.1', 'fasteners', 'numcodecs>=0.10.0', ] @@ -30,7 +30,6 @@ classifiers = [ 'Topic :: Software Development :: Libraries :: Python Modules', 'Operating System :: Unix', 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.8', 'Programming Language :: Python :: 3.9', 'Programming Language :: Python :: 3.10', 'Programming Language :: Python :: 3.11', From 6ad464bb04bffa83b9665dd09caf0f8aaf6b367d Mon Sep 17 00:00:00 2001 From: Joe Hamman Date: Wed, 10 Jan 2024 11:59:46 -0800 Subject: [PATCH 024/130] Add Norman Rzepka to core-dev team (#1630) --- TEAM.md | 1 + 1 file changed, 1 insertion(+) diff --git a/TEAM.md b/TEAM.md index a95885ebe5..6a22d83d1f 100644 --- a/TEAM.md +++ b/TEAM.md @@ -6,6 +6,7 @@ - @d-v-b (Davis Bennett) - @jakirkham (jakirkham) - @martindurant (Martin Durant) +- @normanrz (Norman Rzepka) ## Emeritus core-developers - @alimanfoo (Alistair Miles) From a292dc43f8d0181214ded83124ebd4f85db0ff50 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 15 Jan 2024 11:14:02 -0800 Subject: [PATCH 025/130] chore: update pre-commit hooks (#1633) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit updates: - [github.com/astral-sh/ruff-pre-commit: v0.1.11 → v0.1.13](https://github.com/astral-sh/ruff-pre-commit/compare/v0.1.11...v0.1.13) Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 340366ef53..7d1f9254ae 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -8,7 +8,7 @@ default_language_version: repos: - repo: https://github.com/astral-sh/ruff-pre-commit # Ruff version. - rev: 'v0.1.11' + rev: 'v0.1.13' hooks: - id: ruff - repo: https://github.com/psf/black From 68c87bb51d922487647fa6188392caf8c1d9a83c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 15 Jan 2024 20:33:42 +0100 Subject: [PATCH 026/130] Bump actions/download-artifact from 3 to 4 (#1611) * Bump actions/download-artifact from 3 to 4 Bumps [actions/download-artifact](https://github.com/actions/download-artifact) from 3 to 4. - [Release notes](https://github.com/actions/download-artifact/releases) - [Commits](https://github.com/actions/download-artifact/compare/v3...v4) --- updated-dependencies: - dependency-name: actions/download-artifact dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] * Also bump upload-artifact see https://github.com/actions/download-artifact?tab=readme-ov-file#breaking-changes > Downloading artifacts that were created from action/upload-artifact@v3 and below are not supported. --------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Joe Hamman Co-authored-by: Josh Moore --- .github/workflows/releases.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/releases.yml b/.github/workflows/releases.yml index 31a7e2770c..250c6112c8 100644 --- a/.github/workflows/releases.yml +++ b/.github/workflows/releases.yml @@ -36,7 +36,7 @@ jobs: else echo "All seem good" fi - - uses: actions/upload-artifact@v3 + - uses: actions/upload-artifact@v4 with: name: releases path: dist @@ -45,7 +45,7 @@ jobs: needs: [build_artifacts] runs-on: ubuntu-latest steps: - - uses: actions/download-artifact@v3 + - uses: actions/download-artifact@v4 with: name: releases path: dist @@ -60,7 +60,7 @@ jobs: runs-on: ubuntu-latest if: github.event_name == 'push' && startsWith(github.event.ref, 'refs/tags/v') steps: - - uses: actions/download-artifact@v3 + - uses: actions/download-artifact@v4 with: name: releases path: dist From 1d56da0eb54f64840b1fb0f42c72622233f2f1f6 Mon Sep 17 00:00:00 2001 From: Jeff Peck Date: Tue, 16 Jan 2024 07:00:17 -0500 Subject: [PATCH 027/130] Update tutorial.rst to include section about accessing Zip Files on S3 (#1615) * Update tutorial.rst to include section about accessing Zip Files on S3 Per discussion here, add information about about accessing zip files on s3: https://github.com/zarr-developers/zarr-python/discussions/1613 * Update release.rst * Implement d-v-b's suggestions --------- Co-authored-by: Davis Bennett Co-authored-by: Josh Moore --- docs/release.rst | 2 ++ docs/tutorial.rst | 25 +++++++++++++++++++++++++ 2 files changed, 27 insertions(+) diff --git a/docs/release.rst b/docs/release.rst index a3e0831ba4..ab74a3debd 100644 --- a/docs/release.rst +++ b/docs/release.rst @@ -45,6 +45,8 @@ Docs * Minor tweak to advanced indexing tutorial examples. By :user:`Ross Barnowski ` :issue:`1550`. +* Added section about accessing zip files that are on s3. + By :user:`Jeff Peck ` :issue:`1613`. Maintenance ~~~~~~~~~~~ diff --git a/docs/tutorial.rst b/docs/tutorial.rst index 4099bac1c8..351eef064a 100644 --- a/docs/tutorial.rst +++ b/docs/tutorial.rst @@ -1000,6 +1000,31 @@ separately from Zarr. .. _tutorial_copy: +Accessing Zip Files on S3 +~~~~~~~~~~~~~~~~~~~~~~~~~ + +The built-in `ZipStore` will only work with paths on the local file-system, however +it is also possible to access ``.zarr.zip`` data on the cloud. Here is an example of +accessing a zipped Zarr file on s3: + + >>> s3_path = "s3://path/to/my.zarr.zip" + >>> + >>> s3 = s3fs.S3FileSystem() + >>> f = s3.open(s3_path) + >>> fs = ZipFileSystem(f, mode="r") + >>> store = FSMap("", fs, check=False) + >>> + >>> # cache is optional, but may be a good idea depending on the situation + >>> cache = zarr.storage.LRUStoreCache(store, max_size=2**28) + >>> z = zarr.group(store=cache) + +This store can also be generated with ``fsspec``'s handler chaining, like so: + + >>> store = zarr.storage.FSStore(url=f"zip::{s3_path}", mode="r") + +This can be especially useful if you have a very large ``.zarr.zip`` file on s3 +and only need to access a small portion of it. + Consolidating metadata ~~~~~~~~~~~~~~~~~~~~~~ From 8ac8553f25eb338d6044d1232b4a643036979486 Mon Sep 17 00:00:00 2001 From: Joe Hamman Date: Tue, 16 Jan 2024 09:14:10 -0800 Subject: [PATCH 028/130] doc(v3): add v3 roadmap and design document (#1583) * doc(v3): add v3 roadmap and design document * Update v3-roadmap-and-design.md * updates after latest round of reviews * Update v3-roadmap-and-design.md Co-authored-by: Norman Rzepka * Update v3-roadmap-and-design.md Co-authored-by: Sanket Verma --------- Co-authored-by: Norman Rzepka Co-authored-by: Sanket Verma --- v3-roadmap-and-design.md | 429 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 429 insertions(+) create mode 100644 v3-roadmap-and-design.md diff --git a/v3-roadmap-and-design.md b/v3-roadmap-and-design.md new file mode 100644 index 0000000000..696799e56f --- /dev/null +++ b/v3-roadmap-and-design.md @@ -0,0 +1,429 @@ +# Zarr Python Roadmap + +- Status: draft +- Author: Joe Hamman +- Created On: October 31, 2023 +- Input from: + - Davis Bennett / @d-v-b + - Norman Rzepka / @normanrz + - Deepak Cherian @dcherian + - Brian Davis / @monodeldiablo + - Oliver McCormack / @olimcc + - Ryan Abernathey / @rabernat + - Jack Kelly / @JackKelly + - Martin Durrant / @martindurant + +## Introduction + +This document lays out a design proposal for version 3.0 of the [Zarr-Python](https://zarr.readthedocs.io/en/stable/) package. A specific focus of the design is to bring Zarr-Python's API up to date with the [Zarr V3 specification](https://zarr-specs.readthedocs.io/en/latest/v3/core/v3.0.html), with the hope of enabling the development of the many features and extensions that motivated the V3 Spec. The ideas presented here are expected to result in a major release of Zarr-Python (version 3.0) including significant a number of breaking API changes. +For clarity, “V3” will be used to describe the version of the Zarr specification and “3.0” will be used to describe the release tag of the Zarr-Python project. + +### Current status of V3 in Zarr-Python + +During the development of the V3 Specification, a [prototype implementation](https://github.com/zarr-developers/zarr-python/pull/898) was added to the Zarr-Python library. Since that implementation, the V3 spec evolved in significant ways and as a result, the Zarr-Python library is now out of sync with the approved spec. Downstream libraries (e.g. [Xarray](https://github.com/pydata/xarray)) have added support for this implementation and will need to migrate to the accepted spec when its available in Zarr-Python. + +## Goals + +- Provide a complete implementation of Zarr V3 through the Zarr-Python API +- Clear the way for exciting extensions / ZEPs (i.e. [sharding](https://zarr-specs.readthedocs.io/en/latest/v3/codecs/sharding-indexed/v1.0.html), [variable chunking](https://zarr.dev/zeps/draft/ZEP0003.html), etc.) +- Provide a developer API that can be used to implement and register V3 extensions +- Improve the performance of Zarr-Python by streamlining the interface between the Store layer and higher level APIs (e.g. Groups and Arrays) +- Clean up the internal and user facing APIs +- Improve code quality and robustness (e.g. achieve 100% type hint coverage) +- Align the Zarr-Python array API with the [array API Standard](https://data-apis.org/array-api/latest/) + +## Examples of what 3.0 will enable? +1. Reading and writing V3 spec-compliant groups and arrays +2. V3 extensions including sharding and variable chunking. +3. Improved performance by leveraging concurrency when creating/reading/writing to stores (imagine a `create_hierarchy(zarr_objects)` function). +4. User-developed extensions (e.g. storage-transformers) can be registered with Zarr-Python at runtime + +## Non-goals (of this document) + +- Implementation of any unaccepted Zarr V3 extensions +- Major revisions to the Zarr V3 spec + +## Requirements + +1. Read and write spec compliant V2 and V3 data +2. Limit unnecessary traffic to/from the store +3. Cleanly define the Array/Group/Store abstractions +4. Cleanly define how V2 will be supported going forward +5. Provide a clear roadmap to help users upgrade to 3.0 +6. Developer tools / hooks for registering extensions + +## Design + +### Async API + +Zarr-Python is an IO library. As such, supporting concurrent action against the storage layer is critical to achieving acceptable performance. The Zarr-Python 2 was not designed with asynchronous computation in mind and as a result has struggled to effectively leverage the benefits of concurrency. At one point, `getitems` and `setitems` support was added to the Zarr store model but that is only used for operating on a set of chunks in a single variable. + +With Zarr-Python 3.0, we have the opportunity to revisit this design. The proposal here is as follows: + +1. The `Store` interface will be entirely async. +2. On top of the async `Store` interface, we will provide an `AsyncArray` and `AsyncGroup` interface. +3. Finally, the primary user facing API will be synchronous `Array` and `Group` classes that wrap the async equivalents. + +**Examples** + +- **Store** + + ```python + class Store: + ... + async def get(self, key: str) -> bytes: + ... + async def get_partial_values(self, key_ranges: List[Tuple[str, Tuple[int, Optional[int]]]]) -> bytes: + ... + # (no sync interface here) + ``` +- **Array** + + ```python + class AsyncArray: + ... + + async def getitem(self, selection: Selection) -> np.ndarray: + # the core logic for getitem goes here + + class Array: + _async_array: AsyncArray + + def __getitem__(self, selection: Selection) -> np.ndarray: + return sync(self._async_array.getitem(selection)) + ``` +- **Group** + + ```python + class AsyncGroup: + ... + + async def create_group(self, path: str, **kwargs) -> AsyncGroup: + # the core logic for create_group goes here + + class Group: + _async_group: AsyncGroup + + def create_group(self, path: str, **kwargs) -> Group: + return sync(self._async_group.create_group(path, **kwargs)) + ``` +**Internal Synchronization API** + +With the `Store` and core `AsyncArray`/ `AsyncGroup` classes being predominantly async, Zarr-Python will need an internal API to provide a synchronous API. The proposal here is to use the approach in [fsspec](https://github.com/fsspec/filesystem_spec/blob/master/fsspec/asyn.py) to provide a high-level `sync` function that takes an `awaitable` and runs it in its managed IO Loop / thread. + +**FAQ** +1. Why two levels of Arrays/groups? + a. First, this is an intentional decision and departure from the current Zarrita implementation + b. The idea is that users rarely want to mix interfaces. Either they are working within an async context (currently quite rare) or they are in a typical synchronous context. + c. Splitting the two will allow us to clearly define behavior on the `AsyncObj` and simply wrap it in the `SyncObj`. +2. What if a store is only has a synchronous backend? + a. First off, this is expected to be a fairly rare occurrence. Most storage backends have async interfaces. + b. But in the event a storage backend doesn’t have a async interface, there is nothing wrong with putting synchronous code in `async` methods. There are approaches to enabling concurrent action through wrappers like AsyncIO's `loop.run_in_executor` ([ref 1](https://stackoverflow.com/questions/38865050/is-await-in-python3-cooperative-multitasking ), [ref 2](https://stackoverflow.com/a/43263397/732596), [ref 3](https://bbc.github.io/cloudfit-public-docs/asyncio/asyncio-part-5.html), [ref 4](https://docs.python.org/3/library/asyncio-eventloop.html#asyncio.loop.run_in_executor). +3. Will Zarr help manage the async contexts encouraged by some libraries (e.g. [AioBotoCore](https://aiobotocore.readthedocs.io/en/latest/tutorial.html#using-botocore))? + a. Many async IO libraries require entering an async context before interacting with the API. We expect some experimentation to be needed here but the initial design will follow something close to what fsspec does ([example in s3fs](https://github.com/fsspec/s3fs/blob/949442693ec940b35cda3420c17a864fbe426567/s3fs/core.py#L527)). +4. Why not provide a synchronous Store interface? + a. We could but this design is simpler. It would mean supporting it in the `AsyncGroup` and `AsyncArray` classes which, may be more trouble than its worth. Storage backends that do not have an async API will be encouraged to wrap blocking calls in an async wrapper (e.g. `loop.run_in_executor`). + +### Store API + +The `Store` API is specified directly in the V3 specification. All V3 stores should implement this abstract API, omitting Write and List support as needed. As described above, all stores will be expected to expose the required methods as async methods. + +**Example** + +```python +class ReadWriteStore: + ... + async def get(self, key: str) -> bytes: + ... + + async def get_partial_values(self, key_ranges: List[Tuple[str, int, int]) -> bytes: + ... + + async def set(self, key: str, value: Union[bytes, bytearray, memoryview]) -> None: + ... # required for writable stores + + async def set_partial_values(self, key_start_values: List[Tuple[str, int, Union[bytes, bytearray, memoryview]]]) -> None: + ... # required for writable stores + + async def list(self) -> List[str]: + ... # required for listable stores + + async def list_prefix(self, prefix: str) -> List[str]: + ... # required for listable stores + + async def list_dir(self, prefix: str) -> List[str]: + ... # required for listable stores + + # additional (optional methods) + async def getsize(self, prefix: str) -> int: + ... + + async def rename(self, src: str, dest: str) -> None + ... + +``` + +Recognizing that there are many Zarr applications today that rely on the `MutableMapping` interface supported by Zarr-Python 2, a wrapper store will be developed to allow existing stores to plug directly into this API. + +### Array API + +The user facing array interface will implement a subset of the [Array API Standard](https://data-apis.org/array-api/latest/). Most of the computational parts of the Array API Standard don’t fit into Zarr right now. That’s okay. What matters most is that we ensure we can give downstream applications a compliant API. + +*Note, Zarr already does most of this so this is more about formalizing the relationship than a substantial change in API.* + +| | Included | Not Included | Unknown / Maybe possible? | +| --- | --- | --- | --- | +| Attributes | `dtype` | `mT` | `device` | +| | `ndim` | `T` | | +| | `shape` | | | +| | `size` | | | +| Methods | `__getitem__` | `__array_namespace__` | `to_device` | +| | `__setitem__` | `__abs__` | `__bool__` | +| | `__eq__` | `__add__` | `__complex__` | +| | `__bool__` | `__and__` | `__dlpack__` | +| | | `__floordiv__` | `__dlpack_device__` | +| | | `__ge__` | `__float__` | +| | | `__gt__` | `__index__` | +| | | `__invert__` | `__int__` | +| | | `__le__` | | +| | | `__lshift__` | | +| | | `__lt__` | | +| | | `__matmul__` | | +| | | `__mod__` | | +| | | `__mul__` | | +| | | `__ne__` | | +| | | `__neg__` | | +| | | `__or__` | | +| | | `__pos__` | | +| | | `__pow__` | | +| | | `__rshift__` | | +| | | `__sub__` | | +| | | `__truediv__` | | +| | | `__xor__` | | +| Creation functions (`zarr.creation`) | `zeros` | | `arange` | +| | `zeros_like` | | `asarray` | +| | `ones` | | `eye` | +| | `ones_like` | | `from_dlpack` | +| | `full` | | `linspace` | +| | `full_like` | | `meshgrid` | +| | `empty` | | `tril` | +| | `empty_like` | | `triu` | + +In addition to the core array API defined above, the Array class should have the following Zarr specific properties: + +- `.metadata` (see Metadata Interface below) +- `.attrs` - (pull from metadata object) +- `.info` - (pull from existing property †) + +*† In Zarr-Python 2, the info property lists the store to identify initialized chunks. By default this will be turned off in 3.0 but will be configurable.* + +**Indexing** + +Zarr-Python currently supports `__getitem__` style indexing and the special `oindex` and `vindex` indexers. These are not part of the current Array API standard (see [data-apis/array-api\#669](https://github.com/data-apis/array-api/issues/669)) but they have been [proposed as a NEP](https://numpy.org/neps/nep-0021-advanced-indexing.html). Zarr-Python will maintain these in 3.0. + +We are also exploring a new high-level indexing API that will enabled optimized batch/concurrent loading of many chunks. We expect this to be important to enable performant loading of data in the context of sharding. See [this discussion](https://github.com/zarr-developers/zarr-python/discussions/1569) for more detail. + +Concurrent indexing across multiple arrays will be possible using the AsyncArray API. + +**Async and Sync Array APIs** + +Most the logic to support Zarr Arrays will live in the `AsyncArray` class. There are a few notable differences that should be called out. + +| Sync Method | Async Method | +| --- | --- | +| `__getitem__` | `getitem` | +| `__setitem__` | `setitem` | +| `__eq__` | `equals` | + +**Metadata interface** + +Zarr-Python 2.* closely mirrors the V2 spec metadata schema in the Array and Group classes. In 3.0, we plan to move the underlying metadata representation to a separate interface (e.g. `Array.metadata`). This interface will return either a `V2ArrayMetadata` or `V3ArrayMetadata` object (both will inherit from a parent `ArrayMetadataABC` class. The `V2ArrayMetadata` and `V3ArrayMetadata` classes will be responsible for producing valid JSON representations of their metadata, and yielding a consistent view to the `Array` or `Group` class. + +### Group API + +The main question is how closely we should follow the existing Zarr-Python implementation / `MutableMapping` interface. The table below shows the primary `Group` methods in Zarr-Python 2 and attempts to identify if and how they would be implemented in 3.0. + +| V2 Group Methods | `AsyncGroup` | `Group` | `h5py_compat.Group`` | +| --- | --- | --- | --- | +| `__len__` | `length` | `__len__` | `__len__` | +| `__iter__` | `__aiter__` | `__iter__` | `__iter__` | +| `__contains__` | `contains` | `__contains__` | `__contains__` | +| `__getitem__` | `getitem` | `__getitem__` | `__getitem__` | +| `__enter__` | N/A | N/A | `__enter__` | +| `__exit__` | N/A | N/A | `__exit__` | +| `group_keys` | `group_keys` | `group_keys` | N/A | +| `groups` | `groups` | `groups` | N/A | +| `array_keys` | `array_key` | `array_keys` | N/A | +| `arrays` | `arrays`* | `arrays` | N/A | +| `visit` | ? | ? | `visit` | +| `visitkeys` | ? | ? | ? | +| `visitvalues` | ? | ? | ? | +| `visititems` | ? | ? | `visititems` | +| `tree` | `tree` | `tree` | `Both` | +| `create_group` | `create_group` | `create_group` | `create_group` | +| `require_group` | N/A | N/A | `require_group` | +| `create_groups` | ? | ? | N/A | +| `require_groups` | ? | ? | ? | +| `create_dataset` | N/A | N/A | `create_dataset` | +| `require_dataset` | N/A | N/A | `require_dataset` | +| `create` | `create_array` | `create_array` | N/A | +| `empty` | `empty` | `empty` | N/A | +| `zeros` | `zeros` | `zeros` | N/A | +| `ones` | `ones` | `ones` | N/A | +| `full` | `full` | `full` | N/A | +| `array` | `create_array` | `create_array` | N/A | +| `empty_like` | `empty_like` | `empty_like` | N/A | +| `zeros_like` | `zeros_like` | `zeros_like` | N/A | +| `ones_like` | `ones_like` | `ones_like` | N/A | +| `full_like` | `full_like` | `full_like` | N/A | +| `move` | `move` | `move` | `move` | + +**`zarr.h5compat.Group`** + +Zarr-Python 2.* made an attempt to align its API with that of [h5py](https://docs.h5py.org/en/stable/index.html). With 3.0, we will relax this alignment in favor of providing an explicit compatibility module (`zarr.h5py_compat`). This module will expose the `Group` and `Dataset` APIs that map to Zarr-Python’s `Group` and `Array` objects. + +### Creation API + +Zarr-Python 2.* bundles together the creation and serialization of Zarr objects. Zarr-Python 3.* will make it possible to create objects in memory separate from serializing them. This will specifically enable writing hierarchies of Zarr objects in a single batch step. For example: + +```python + +arr1 = Array(shape=(10, 10), path="foo/bar", dtype="i4", store=store) +arr2 = Array(shape=(10, 10), path="foo/spam", dtype="f8", store=store) + +arr1.save() +arr2.save() + +# or equivalently + +zarr.save_many([arr1 ,arr2]) +``` + +*Note: this batch creation API likely needs additional design effort prior to implementation.* + +### Plugin API + +Zarr V3 was designed to be extensible at multiple layers. Zarr-Python will support these extensions through a combination of [Abstract Base Classes](https://docs.python.org/3/library/abc.html) (ABCs) and [Entrypoints](https://packaging.python.org/en/latest/specifications/entry-points/). + +**ABCs** + +Zarr V3 will expose Abstract base classes for the following objects: + +- `Store`, `ReadStore`, `ReadWriteStore`, `ReadListStore`, and `ReadWriteListStore` +- `BaseArray`, `SynchronousArray`, and `AsynchronousArray` +- `BaseGroup`, `SynchronousGroup`, and `AsynchronousGroup` +- `Codec`, `ArrayArrayCodec`, `ArrayBytesCodec`, `BytesBytesCodec` + +**Entrypoints** + +Lots more thinking here but the idea here is to provide entrypoints for `data type`, `chunk grid`, `chunk key encoding`, `codecs`, `storage_transformers` and `stores`. These might look something like: + +``` +entry_points=""" + [zarr.codecs] + blosc_codec=codec_plugin:make_blosc_codec + zlib_codec=codec_plugin:make_zlib_codec +""" +``` + +### Python type hints and static analysis + +Target 100% Mypy coverage in 3.0 source. + +### Observability + +A persistent problem in Zarr-Python is diagnosing problems that span many parts of the stack. To address this in 3.0, we will add a basic logging framework that can be used to debug behavior at various levels of the stack. We propose to add the separate loggers for the following namespaces: + +- `array` +- `group` +- `store` +- `codec` + +These should be documented such that users know how to activate them and developers know how to use them when developing extensions. + +### Dependencies + +Today, Zarr-Python has the following required dependencies: + +```python +dependencies = [ + 'asciitree', + 'numpy>=1.20,!=1.21.0', + 'fasteners', + 'numcodecs>=0.10.0', +] +``` + +What other dependencies should be considered? + +1. Attrs - Zarrita makes extensive use of the Attrs library +2. Fsspec - Zarrita has a hard dependency on Fsspec. This could be easily relaxed though. + +## Breaking changes relative to Zarr-Python 2.* + +1. H5py compat moved to a stand alone module? +2. `Group.__getitem__` support moved to `Group.members.__getitem__`? +3. Others? + +## Open questions + +1. How to treat V2 + a. Note: Zarrita currently implements a separate `V2Array` and `V3Array` classes. This feels less than ideal. + b. We could easily convert metadata from v2 to the V3 Array, but what about writing? + c. Ideally, we don’t have completely separate code paths. But if its too complicated to support both within one interface, its probably better. +2. How and when to remove the current implementation of V3. + a. It's hidden behind a hard-to-use feature flag so we probably don't need to do anything. +4. How to model runtime configuration? +5. Which extensions belong in Zarr-Python and which belong in separate packages? + a. We don't need to take a strong position on this here. It's likely that someone will want to put Sharding in. That will be useful to develop in parallel because it will give us a good test case for the plugin interface. + +## Testing + +Zarr-python 3.0 adds a major new dimension to Zarr: Async support. This also comes with a compatibility risk, we will need to thoroughly test support in key execution environments. Testing plan: +- Reuse the existing test suite for testing the `v3` API. + - `xfail` tests that expose breaking changes with `3.0 - breaking change` description. This will help identify additional and/or unintentional breaking changes + - Rework tests that were only testing internal APIs. +- Add a set of functional / integration tests targeting real-world workflows in various contexts (e.g. w/ Dask) + +## Development process + +Zarr-Python 3.0 will introduce a number of new APIs and breaking changes to existing APIs. In order to facilitate ongoing support for Zarr-Python 2.*, we will take on the following development process: + +- Create a `v3` branch that can be use for developing the core functionality apart from the `main` branch. This will allow us to support ongoing work and bug fixes on the `main` branch. +- Put the `3.0` APIs inside a `zarr.v3` module. Imports from this namespace will all be new APIs that users can develop and test against once the `v3` branch is merged to `main`. +- Kickstart the process by pulling in the current state of `zarrita` - which has many of the features described in this design. +- Release a series of 2.* releases with the `v3` namespace +- When `v3` is complete, move contents of `v3` to the package root + +**Milestones** + +Below are a set of specific milestones leading toward the completion of this process. As work begins, we expect this list to grow in specificity. + +1. Port current version of Zarrita to Zarr-Python +2. Formalize Async interface by splitting `Array` and `Group` objects into Sync and Async versions +4. Implement "fancy" indexing operations on the `AsyncArray` +6. Implement an abstract base class for the `Store` interface and a wrapper `Store` to make use of existing `MutableMapping` stores. +7. Rework the existing unit test suite to use the `v3` namespace. +8. Develop a plugin interface for extensions +9. Develop a set of functional and integration tests +10. Work with downstream libraries (Xarray, Dask, etc.) to test new APIs + +## TODOs + +The following subjects are not covered in detail above but perhaps should be. Including them here so they are not forgotten. + +1. [Store] Should Zarr provide an API for caching objects after first read/list/etc. Read only stores? +2. [Array] buffer protocol support +3. [Array] `meta_array` support +4. [Extensions] Define how Zarr-Python will consume the various plugin types +5. [Misc] H5py compatibility requires a bit more work and a champion to drive it forward. +6. [Misc] Define `chunk_store` API in 3.0 +7. [Misc] Define `synchronizer` API in 3.0 + +## References + +1. [Zarr-Python repository](https://github.com/zarr-developers/zarr-python) +2. [Zarr core specification (version 3.0) — Zarr specs documentation](https://zarr-specs.readthedocs.io/en/latest/v3/core/v3.0.html#) +3. [Zarrita repository](https://github.com/scalableminds/zarrita) +4. [Async-Zarr](https://github.com/martindurant/async-zarr) +5. [Zarr-Python Discussion Topic](https://github.com/zarr-developers/zarr-python/discussions/1569) From a81db0782535ba04c32c277102a6457d118a73e8 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 22 Jan 2024 11:50:32 -0800 Subject: [PATCH 029/130] chore: update pre-commit hooks (#1636) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit updates: - [github.com/astral-sh/ruff-pre-commit: v0.1.13 → v0.1.14](https://github.com/astral-sh/ruff-pre-commit/compare/v0.1.13...v0.1.14) Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 7d1f9254ae..a7f48d7cd6 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -8,7 +8,7 @@ default_language_version: repos: - repo: https://github.com/astral-sh/ruff-pre-commit # Ruff version. - rev: 'v0.1.13' + rev: 'v0.1.14' hooks: - id: ruff - repo: https://github.com/psf/black From 4f2ace4b8708cf91f4bd29ae3ed210a3e66f235c Mon Sep 17 00:00:00 2001 From: Hood Chatham Date: Wed, 14 Feb 2024 04:09:41 -0800 Subject: [PATCH 030/130] Fix zarr sync (#1663) This patch removes fasteners and disables zarr.sync which uses process and thread Co-authored-by: Wei Ouyang --- pyproject.toml | 2 +- zarr/sync.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index a85e49e82c..4da3079808 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -14,7 +14,7 @@ requires-python = ">=3.9" dependencies = [ 'asciitree', 'numpy>=1.21.1', - 'fasteners', + 'fasteners; sys_platform != "emscripten"', 'numcodecs>=0.10.0', ] dynamic = [ diff --git a/zarr/sync.py b/zarr/sync.py index 2e843f6557..03046a4a32 100644 --- a/zarr/sync.py +++ b/zarr/sync.py @@ -3,8 +3,6 @@ from threading import Lock from typing import Protocol -import fasteners - class Synchronizer(Protocol): """Base class for synchronizers.""" @@ -49,6 +47,8 @@ def __init__(self, path): self.path = path def __getitem__(self, item): + import fasteners + path = os.path.join(self.path, item) lock = fasteners.InterProcessLock(path) return lock From 0b0ac8857a52653fdb500cc9da7b51b0ec8a05b5 Mon Sep 17 00:00:00 2001 From: Sanket Verma Date: Thu, 15 Feb 2024 01:47:27 +0530 Subject: [PATCH 031/130] Update release.rst (#1621) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Update release.rst * Update release.rst * Change 2.16.2 → 2.17.0 * Update moto for test_s3 * Skip bsddb3 tests to prevent warning failure * Fix more user warning tests * Fix even more user warning tests * Skip coverage for importorskips * Move to have_X skip method for deps * Update release.rst (PR#1663) * Fix test_core.py 'compile' issues * Add black formatting * Drop Windows/3.9 build due to unrelated failures * fix typo --------- Co-authored-by: Davis Bennett Co-authored-by: Josh Moore --- .github/workflows/windows-testing.yml | 2 +- docs/release.rst | 40 +++++++++++++++++++++++++++ requirements_dev_optional.txt | 2 +- zarr/tests/test_core.py | 28 +++++++++++++------ zarr/tests/test_storage.py | 2 +- zarr/tests/util.py | 24 ++++++++++++++++ 6 files changed, 87 insertions(+), 11 deletions(-) diff --git a/.github/workflows/windows-testing.yml b/.github/workflows/windows-testing.yml index 5c3252c0ba..0ef7f21758 100644 --- a/.github/workflows/windows-testing.yml +++ b/.github/workflows/windows-testing.yml @@ -16,7 +16,7 @@ jobs: strategy: fail-fast: True matrix: - python-version: ['3.9', '3.10', '3.11'] + python-version: ['3.10', '3.11'] steps: - uses: actions/checkout@v4 with: diff --git a/docs/release.rst b/docs/release.rst index ab74a3debd..0f199aadd2 100644 --- a/docs/release.rst +++ b/docs/release.rst @@ -18,12 +18,20 @@ Release notes Unreleased ---------- +.. _release_2.17.0: + +2.17.0 +------ + Enhancements ~~~~~~~~~~~~ * Added type hints to ``zarr.creation.create()``. By :user:`David Stansby ` :issue:`1536`. +* Pyodide support: Don't require fasteners on Emscripten. + By :user:`Hood Chatham ` :issue:`1663`. + Docs ~~~~ @@ -45,9 +53,21 @@ Docs * Minor tweak to advanced indexing tutorial examples. By :user:`Ross Barnowski ` :issue:`1550`. +* Automatically document array members using sphinx-automodapi. + By :user:`David Stansby ` :issue:`1547`. + +* Add a markdown file documenting the current and former core-developer team. + By :user:`Joe Hamman ` :issue:`1628`. + +* Add Norman Rzepka to core-dev team. + By :user:`Joe Hamman ` :issue:`1630`. + * Added section about accessing zip files that are on s3. By :user:`Jeff Peck ` :issue:`1613`. +* Add V3 roadmap and design document. + By :user:`Joe Hamman ` :issue:`1583`. + Maintenance ~~~~~~~~~~~ @@ -75,6 +95,26 @@ Maintenance * Remove ``sphinx-rtd-theme`` dependency from ``pyproject.toml``. By :user:`Sanket Verma ` :issue:`1563`. +* Remove ``CODE_OF_CONDUCT.md`` file from the Zarr-Python repository. + By :user:`Sanket Verma ` :issue:`1572`. + +* Bump version of black in pre-commit. + By :user:`David Stansby ` :issue:`1559`. + +* Use list comprehension where applicable. + By :user:`Dimitri Papadopoulos Orfanos ` :issue:`1555`. + +* Use format specification mini-language to format string. + By :user:`Dimitri Papadopoulos Orfanos ` :issue:`1558`. + +* Single startswith() call instead of multiple ones. + By :user:`Dimitri Papadopoulos Orfanos ` :issue:`1556`. + +* Move codespell options around. + By :user:`Dimitri Papadopoulos Orfanos ` :issue:`1196`. + +* Remove unused mypy ignore comments. + By :user:`David Stansby ` :issue:`1602`. .. _release_2.16.1: diff --git a/requirements_dev_optional.txt b/requirements_dev_optional.txt index b4de5fd515..d1ee5a891d 100644 --- a/requirements_dev_optional.txt +++ b/requirements_dev_optional.txt @@ -20,4 +20,4 @@ pytest-timeout==2.2.0 h5py==3.10.0 fsspec==2023.12.2 s3fs==2023.12.2 -moto[server]>=4.0.8 +moto[server]>=5.0.1 diff --git a/zarr/tests/test_core.py b/zarr/tests/test_core.py index a3fde4050d..cf15703497 100644 --- a/zarr/tests/test_core.py +++ b/zarr/tests/test_core.py @@ -73,7 +73,15 @@ ) from zarr.tests.test_storage_v3 import DummyStorageTransfomer from zarr.util import buffer_size -from zarr.tests.util import abs_container, skip_test_env_var, have_fsspec, mktemp +from zarr.tests.util import ( + abs_container, + have_bsddb3, + have_fsspec, + have_lmdb, + have_sqlite3, + mktemp, + skip_test_env_var, +) from zarr.types import DIMENSION_SEPARATOR # noinspection PyMethodMayBeStatic @@ -2038,9 +2046,11 @@ def test_nbytes_stored(self): pass # not implemented +@pytest.mark.skipif(have_bsddb3 is False, reason="needs bsddb3") class TestArrayWithDBMStoreBerkeleyDB(TestArray): def create_store(self): - bsddb3 = pytest.importorskip("bsddb3") + import bsddb3 + path = mktemp(suffix=".dbm") atexit.register(os.remove, path) store = DBMStore(path, flag="n", open=bsddb3.btopen) @@ -2050,9 +2060,9 @@ def test_nbytes_stored(self): pass # not implemented +@pytest.mark.skipif(have_lmdb is False, reason="needs lmdb") class TestArrayWithLMDBStore(TestArray): def create_store(self): - pytest.importorskip("lmdb") path = mktemp(suffix=".lmdb") atexit.register(atexit_rmtree, path) store = LMDBStore(path, buffers=True) @@ -2065,9 +2075,9 @@ def test_nbytes_stored(self): pass # not implemented +@pytest.mark.skipif(have_lmdb is False, reason="needs lmdb") class TestArrayWithLMDBStoreNoBuffers(TestArray): def create_store(self): - pytest.importorskip("lmdb") path = mktemp(suffix=".lmdb") atexit.register(atexit_rmtree, path) store = LMDBStore(path, buffers=False) @@ -2077,9 +2087,9 @@ def test_nbytes_stored(self): pass # not implemented +@pytest.mark.skipif(have_sqlite3 is False, reason="needs sqlite3") class TestArrayWithSQLiteStore(TestArray): def create_store(self): - pytest.importorskip("sqlite3") path = mktemp(suffix=".db") atexit.register(atexit_rmtree, path) store = SQLiteStore(path) @@ -2758,9 +2768,11 @@ def test_nbytes_stored(self): @pytest.mark.skipif(not v3_api_available, reason="V3 is disabled") +@pytest.mark.skipif(have_bsddb3 is False, reason="needs bsddb3") class TestArrayWithDBMStoreV3BerkeleyDB(TestArrayV3): def create_store(self) -> DBMStoreV3: - bsddb3 = pytest.importorskip("bsddb3") + import bsddb3 + path = mktemp(suffix=".dbm") atexit.register(os.remove, path) store = DBMStoreV3(path, flag="n", open=bsddb3.btopen) @@ -2771,11 +2783,11 @@ def test_nbytes_stored(self): @pytest.mark.skipif(not v3_api_available, reason="V3 is disabled") +@pytest.mark.skipif(have_lmdb is False, reason="needs lmdb") class TestArrayWithLMDBStoreV3(TestArrayV3): lmdb_buffers = True def create_store(self) -> LMDBStoreV3: - pytest.importorskip("lmdb") path = mktemp(suffix=".lmdb") atexit.register(atexit_rmtree, path) store = LMDBStoreV3(path, buffers=self.lmdb_buffers) @@ -2797,9 +2809,9 @@ def test_nbytes_stored(self): @pytest.mark.skipif(not v3_api_available, reason="V3 is disabled") +@pytest.mark.skipif(have_sqlite3 is False, reason="needs sqlite3") class TestArrayWithSQLiteStoreV3(TestArrayV3): def create_store(self): - pytest.importorskip("sqlite3") path = mktemp(suffix=".db") atexit.register(atexit_rmtree, path) store = SQLiteStoreV3(path) diff --git a/zarr/tests/test_storage.py b/zarr/tests/test_storage.py index 25863749d8..e4e3d93f5f 100644 --- a/zarr/tests/test_storage.py +++ b/zarr/tests/test_storage.py @@ -1396,7 +1396,7 @@ def s3(request): port = 5555 endpoint_uri = "http://127.0.0.1:%d/" % port proc = subprocess.Popen( - shlex.split("moto_server s3 -p %d" % port), + shlex.split("moto_server -p %d" % port), stderr=subprocess.DEVNULL, stdout=subprocess.DEVNULL, ) diff --git a/zarr/tests/util.py b/zarr/tests/util.py index b4f00f703d..b3c3249cab 100644 --- a/zarr/tests/util.py +++ b/zarr/tests/util.py @@ -69,6 +69,30 @@ def skip_test_env_var(name): have_fsspec = False +try: + import bsddb3 # noqa: F401 + + have_bsddb3 = True +except ImportError: # pragma: no cover + have_bsddb3 = False + + +try: + import lmdb # noqa: F401 + + have_lmdb = True +except ImportError: # pragma: no cover + have_lmdb = False + + +try: + import sqlite3 # noqa: F401 + + have_sqlite3 = True +except ImportError: # pragma: no cover + have_sqlite3 = False + + def abs_container(): from azure.core.exceptions import ResourceExistsError import azure.storage.blob as asb From e50b47196eb4e4071158baba22567713ad012837 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 14 Feb 2024 22:04:51 +0100 Subject: [PATCH 032/130] Bump numpy from 1.24.3 to 1.26.1 (#1543) Bumps [numpy](https://github.com/numpy/numpy) from 1.24.3 to 1.26.1. - [Release notes](https://github.com/numpy/numpy/releases) - [Changelog](https://github.com/numpy/numpy/blob/main/doc/RELEASE_WALKTHROUGH.rst) - [Commits](https://github.com/numpy/numpy/compare/v1.24.3...v1.26.1) --- updated-dependencies: - dependency-name: numpy dependency-type: direct:development update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Davis Bennett Co-authored-by: Josh Moore Co-authored-by: Joe Hamman --- requirements_dev_numpy.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements_dev_numpy.txt b/requirements_dev_numpy.txt index a6135bd831..c8c5f7d7ab 100644 --- a/requirements_dev_numpy.txt +++ b/requirements_dev_numpy.txt @@ -1,4 +1,4 @@ # Break this out into a separate file to allow testing against # different versions of numpy. This file should pin to the latest # numpy version. -numpy==1.24.3 +numpy==1.26.1 From 81bbb2e7f28d64335d835523057041f11cdc7843 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 14 Feb 2024 22:05:20 +0100 Subject: [PATCH 033/130] chore: update pre-commit hooks (#1642) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * chore: update pre-commit hooks updates: - [github.com/astral-sh/ruff-pre-commit: v0.1.14 → v0.2.1](https://github.com/astral-sh/ruff-pre-commit/compare/v0.1.14...v0.2.1) - [github.com/psf/black: 23.12.1 → 24.2.0](https://github.com/psf/black/compare/23.12.1...24.2.0) * run black incl. comments for '...' --------- Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Co-authored-by: Josh Moore --- .pre-commit-config.yaml | 4 ++-- zarr/convenience.py | 1 + zarr/core.py | 8 +++++--- zarr/indexing.py | 10 +++++----- zarr/n5.py | 1 + zarr/storage.py | 1 + zarr/sync.py | 1 + zarr/types.py | 1 + 8 files changed, 17 insertions(+), 10 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index a7f48d7cd6..c7d4f32c68 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -8,11 +8,11 @@ default_language_version: repos: - repo: https://github.com/astral-sh/ruff-pre-commit # Ruff version. - rev: 'v0.1.14' + rev: 'v0.2.1' hooks: - id: ruff - repo: https://github.com/psf/black - rev: 23.12.1 + rev: 24.2.0 hooks: - id: black - repo: https://github.com/codespell-project/codespell diff --git a/zarr/convenience.py b/zarr/convenience.py index 9c0deeea47..b4b8bb5293 100644 --- a/zarr/convenience.py +++ b/zarr/convenience.py @@ -1,4 +1,5 @@ """Convenience functions for storing and loading data.""" + import itertools import os import re diff --git a/zarr/core.py b/zarr/core.py index d22a9d79c3..5727afa884 100644 --- a/zarr/core.py +++ b/zarr/core.py @@ -2060,9 +2060,11 @@ def _process_chunk( index_selection = PartialChunkIterator(chunk_selection, self.chunks) for start, nitems, partial_out_selection in index_selection: expected_shape = [ - len(range(*partial_out_selection[i].indices(self.chunks[0] + 1))) - if i < len(partial_out_selection) - else dim + ( + len(range(*partial_out_selection[i].indices(self.chunks[0] + 1))) + if i < len(partial_out_selection) + else dim + ) for i, dim in enumerate(self.chunks) ] if isinstance(cdata, UncompressedPartialReadBufferV3): diff --git a/zarr/indexing.py b/zarr/indexing.py index 3042147ebb..5a2b7c0eb4 100644 --- a/zarr/indexing.py +++ b/zarr/indexing.py @@ -545,11 +545,11 @@ def ix_(selection, shape): # replace slice and int as these are not supported by numpy.ix_ selection = [ - slice_to_range(dim_sel, dim_len) - if isinstance(dim_sel, slice) - else [dim_sel] - if is_integer(dim_sel) - else dim_sel + ( + slice_to_range(dim_sel, dim_len) + if isinstance(dim_sel, slice) + else [dim_sel] if is_integer(dim_sel) else dim_sel + ) for dim_sel, dim_len in zip(selection, shape) ] diff --git a/zarr/n5.py b/zarr/n5.py index 44b44e69e2..c50c18f718 100644 --- a/zarr/n5.py +++ b/zarr/n5.py @@ -1,5 +1,6 @@ """This module contains a storage class and codec to support the N5 format. """ + import os import struct import sys diff --git a/zarr/storage.py b/zarr/storage.py index aa27e98e6f..a26dc636db 100644 --- a/zarr/storage.py +++ b/zarr/storage.py @@ -14,6 +14,7 @@ path) and a `getsize` method (return the size in bytes of a given value). """ + import atexit import errno import glob diff --git a/zarr/sync.py b/zarr/sync.py index 03046a4a32..ba1c5df5b3 100644 --- a/zarr/sync.py +++ b/zarr/sync.py @@ -8,6 +8,7 @@ class Synchronizer(Protocol): """Base class for synchronizers.""" def __getitem__(self, item): + # see subclasses ... diff --git a/zarr/types.py b/zarr/types.py index 1de270f25c..cc29a350f5 100644 --- a/zarr/types.py +++ b/zarr/types.py @@ -10,4 +10,5 @@ class MetaArray(Protocol): def __array_function__(self, func, types, args, kwargs): + # To be extended ... From 367848836535e02eecd92a11ef734dd944285615 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 14 Feb 2024 22:05:40 +0100 Subject: [PATCH 034/130] Bump ipywidgets from 8.1.0 to 8.1.1 (#1538) Bumps [ipywidgets](https://github.com/jupyter-widgets/ipywidgets) from 8.1.0 to 8.1.1. - [Release notes](https://github.com/jupyter-widgets/ipywidgets/releases) - [Commits](https://github.com/jupyter-widgets/ipywidgets/compare/8.1.0...8.1.1) --- updated-dependencies: - dependency-name: ipywidgets dependency-type: direct:development update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Davis Bennett Co-authored-by: Josh Moore --- requirements_dev_optional.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements_dev_optional.txt b/requirements_dev_optional.txt index d1ee5a891d..0ac4922ce1 100644 --- a/requirements_dev_optional.txt +++ b/requirements_dev_optional.txt @@ -3,7 +3,7 @@ lmdb==1.4.1; sys_platform != 'win32' # optional library requirements for Jupyter ipytree==0.2.2 -ipywidgets==8.1.0 +ipywidgets==8.1.1 # optional library requirements for services # don't let pyup change pinning for azure-storage-blob, need to pin to older # version to get compatibility with azure storage emulator on appveyor (FIXME) From 720bea687b444b2082638eb7edc3bb6a4f8fa805 Mon Sep 17 00:00:00 2001 From: Dimitri Papadopoulos Orfanos <3234522+DimitriPapadopoulos@users.noreply.github.com> Date: Wed, 14 Feb 2024 22:14:43 +0100 Subject: [PATCH 035/130] Proper argument for numpy.reshape (#1425) `numpy.reshape` not only accepts a tuple of ints, but also a simple int. Besides `(10)` is not a tuple and is identical to `10`, unlike `(10,)`. --- zarr/tests/test_indexing.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/zarr/tests/test_indexing.py b/zarr/tests/test_indexing.py index af046e9d28..a3afc101c5 100644 --- a/zarr/tests/test_indexing.py +++ b/zarr/tests/test_indexing.py @@ -1632,7 +1632,7 @@ def test_set_selections_with_fields(): ), ( (slice(0, 10, 1),), - np.arange(0, 10).reshape((10)), + np.arange(0, 10).reshape(10), [(0, 10, (slice(0, 10, 1),))], ), ((0,), np.arange(0, 100).reshape((10, 10)), [(0, 10, (slice(0, 1, 1),))]), @@ -1644,7 +1644,7 @@ def test_set_selections_with_fields(): np.arange(0, 100).reshape((10, 10)), [(0, 1, (slice(0, 1, 1), slice(0, 1, 1)))], ), - ((0,), np.arange(0, 10).reshape((10)), [(0, 1, (slice(0, 1, 1),))]), + ((0,), np.arange(0, 10).reshape(10), [(0, 1, (slice(0, 1, 1),))]), pytest.param( (slice(5, 8, 1), slice(2, 4, 1), slice(0, 5, 1)), np.arange(2, 100002).reshape((10, 1, 10000)), From 74498538c180855172573f2983207f74674cbc1c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 15 Feb 2024 09:22:59 +0100 Subject: [PATCH 036/130] Bump ipywidgets from 8.1.1 to 8.1.2 (#1666) Bumps [ipywidgets](https://github.com/jupyter-widgets/ipywidgets) from 8.1.1 to 8.1.2. - [Release notes](https://github.com/jupyter-widgets/ipywidgets/releases) - [Commits](https://github.com/jupyter-widgets/ipywidgets/compare/8.1.1...8.1.2) --- updated-dependencies: - dependency-name: ipywidgets dependency-type: direct:development update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- requirements_dev_optional.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements_dev_optional.txt b/requirements_dev_optional.txt index 0ac4922ce1..e94b814173 100644 --- a/requirements_dev_optional.txt +++ b/requirements_dev_optional.txt @@ -3,7 +3,7 @@ lmdb==1.4.1; sys_platform != 'win32' # optional library requirements for Jupyter ipytree==0.2.2 -ipywidgets==8.1.1 +ipywidgets==8.1.2 # optional library requirements for services # don't let pyup change pinning for azure-storage-blob, need to pin to older # version to get compatibility with azure storage emulator on appveyor (FIXME) From 3db41760e18fb0a69b5066e8c7aba9752a8c474e Mon Sep 17 00:00:00 2001 From: Davis Bennett Date: Thu, 15 Feb 2024 10:54:23 +0100 Subject: [PATCH 037/130] docs: ZIP-related tweaks (#1641) * docs: use 'ZIP archive' instead of 'zip file'; clarify utility of caching in s3 + ZIP example; style * docs: update release notes, correct spelling of greg lee's name in past release notes, and fix markup in past release notes * docs: use 'ZIP archive' instead of 'zip file'; clarify utility of caching in s3 + ZIP example; style * docs: update release notes, correct spelling of greg lee's name in past release notes, and fix markup in past release notes --- docs/release.rst | 20 ++++++++++---------- docs/tutorial.rst | 27 ++++++++++++++------------- 2 files changed, 24 insertions(+), 23 deletions(-) diff --git a/docs/release.rst b/docs/release.rst index 0f199aadd2..b73dcec34f 100644 --- a/docs/release.rst +++ b/docs/release.rst @@ -62,8 +62,8 @@ Docs * Add Norman Rzepka to core-dev team. By :user:`Joe Hamman ` :issue:`1630`. -* Added section about accessing zip files that are on s3. - By :user:`Jeff Peck ` :issue:`1613`. +* Added section about accessing ZIP archives on s3. + By :user:`Jeff Peck ` :issue:`1613`, :issue:`1615`, and :user:`Davis Bennett ` :issue:`1641`. * Add V3 roadmap and design document. By :user:`Joe Hamman ` :issue:`1583`. @@ -157,10 +157,10 @@ Maintenance By :user:`Davis Bennett ` :issue:`1462`. * Style the codebase with ``ruff`` and ``black``. - By :user:`Davis Bennett` :issue:`1459` + By :user:`Davis Bennett ` :issue:`1459` * Ensure that chunks is tuple of ints upon array creation. - By :user:`Philipp Hanslovsky` :issue:`1461` + By :user:`Philipp Hanslovsky ` :issue:`1461` .. _release_2.15.0: @@ -548,7 +548,7 @@ Maintenance By :user:`Saransh Chopra ` :issue:`1079`. * Remove option to return None from _ensure_store. - By :user:`Greggory Lee ` :issue:`1068`. + By :user:`Gregory Lee ` :issue:`1068`. * Fix a typo of "integers". By :user:`Richard Scott ` :issue:`1056`. @@ -566,7 +566,7 @@ Enhancements Since the format is not yet finalized, the classes and functions are not automatically imported into the regular `zarr` name space. Setting the `ZARR_V3_EXPERIMENTAL_API` environment variable will activate them. - By :user:`Greggory Lee `; :issue:`898`, :issue:`1006`, and :issue:`1007` + By :user:`Gregory Lee `; :issue:`898`, :issue:`1006`, and :issue:`1007` as well as by :user:`Josh Moore ` :issue:`1032`. * **Create FSStore from an existing fsspec filesystem**. If you have created @@ -688,7 +688,7 @@ Enhancements higher-level array creation and convenience functions still accept plain Python dicts or other mutable mappings for the ``store`` argument, but will internally convert these to a ``KVStore``. - By :user:`Greggory Lee `; :issue:`839`, :issue:`789`, and :issue:`950`. + By :user:`Gregory Lee `; :issue:`839`, :issue:`789`, and :issue:`950`. * Allow to assign array ``fill_values`` and update metadata accordingly. By :user:`Ryan Abernathey `, :issue:`662`. @@ -835,7 +835,7 @@ Bug fixes ~~~~~~~~~ * Fix FSStore.listdir behavior for nested directories. - By :user:`Greggory Lee `; :issue:`802`. + By :user:`Gregory Lee `; :issue:`802`. .. _release_2.9.4: @@ -919,7 +919,7 @@ Bug fixes By :user:`Josh Moore `; :issue:`781`. * avoid NumPy 1.21.0 due to https://github.com/numpy/numpy/issues/19325 - By :user:`Greggory Lee `; :issue:`791`. + By :user:`Gregory Lee `; :issue:`791`. Maintenance ~~~~~~~~~~~ @@ -931,7 +931,7 @@ Maintenance By :user:`Elliott Sales de Andrade `; :issue:`799`. * TST: add missing assert in test_hexdigest. - By :user:`Greggory Lee `; :issue:`801`. + By :user:`Gregory Lee `; :issue:`801`. .. _release_2.8.3: diff --git a/docs/tutorial.rst b/docs/tutorial.rst index 351eef064a..1f7accab3a 100644 --- a/docs/tutorial.rst +++ b/docs/tutorial.rst @@ -774,7 +774,7 @@ the following code:: Any other compatible storage class could be used in place of :class:`zarr.storage.DirectoryStore` in the code examples above. For example, -here is an array stored directly into a Zip file, via the +here is an array stored directly into a ZIP archive, via the :class:`zarr.storage.ZipStore` class:: >>> store = zarr.ZipStore('data/example.zip', mode='w') @@ -798,12 +798,12 @@ Re-open and check that data have been written:: [42, 42, 42, ..., 42, 42, 42]], dtype=int32) >>> store.close() -Note that there are some limitations on how Zip files can be used, because items -within a Zip file cannot be updated in place. This means that data in the array +Note that there are some limitations on how ZIP archives can be used, because items +within a ZIP archive cannot be updated in place. This means that data in the array should only be written once and write operations should be aligned with chunk boundaries. Note also that the ``close()`` method must be called after writing any data to the store, otherwise essential records will not be written to the -underlying zip file. +underlying ZIP archive. Another storage alternative is the :class:`zarr.storage.DBMStore` class, added in Zarr version 2.2. This class allows any DBM-style database to be used for @@ -846,7 +846,7 @@ respectively require the `redis-py `_ and `pymongo `_ packages to be installed. For compatibility with the `N5 `_ data format, Zarr also provides -an N5 backend (this is currently an experimental feature). Similar to the zip storage class, an +an N5 backend (this is currently an experimental feature). Similar to the ZIP storage class, an :class:`zarr.n5.N5Store` can be instantiated directly:: >>> store = zarr.N5Store('data/example.n5') @@ -1000,12 +1000,13 @@ separately from Zarr. .. _tutorial_copy: -Accessing Zip Files on S3 -~~~~~~~~~~~~~~~~~~~~~~~~~ +Accessing ZIP archives on S3 +~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -The built-in `ZipStore` will only work with paths on the local file-system, however -it is also possible to access ``.zarr.zip`` data on the cloud. Here is an example of -accessing a zipped Zarr file on s3: +The built-in :class:`zarr.storage.ZipStore` will only work with paths on the local file-system; however +it is possible to access ZIP-archived Zarr data on the cloud via the `ZipFileSystem `_ +class from ``fsspec``. The following example demonstrates how to access +a ZIP-archived Zarr group on s3 using `s3fs `_ and ``ZipFileSystem``: >>> s3_path = "s3://path/to/my.zarr.zip" >>> @@ -1014,7 +1015,7 @@ accessing a zipped Zarr file on s3: >>> fs = ZipFileSystem(f, mode="r") >>> store = FSMap("", fs, check=False) >>> - >>> # cache is optional, but may be a good idea depending on the situation + >>> # caching may improve performance when repeatedly reading the same data >>> cache = zarr.storage.LRUStoreCache(store, max_size=2**28) >>> z = zarr.group(store=cache) @@ -1022,7 +1023,7 @@ This store can also be generated with ``fsspec``'s handler chaining, like so: >>> store = zarr.storage.FSStore(url=f"zip::{s3_path}", mode="r") -This can be especially useful if you have a very large ``.zarr.zip`` file on s3 +This can be especially useful if you have a very large ZIP-archived Zarr array or group on s3 and only need to access a small portion of it. Consolidating metadata @@ -1161,7 +1162,7 @@ re-compression, and so should be faster. E.g.:: └── spam (100,) int64 >>> new_root['foo/bar/baz'][:] array([ 0, 1, 2, ..., 97, 98, 99]) - >>> store2.close() # zip stores need to be closed + >>> store2.close() # ZIP stores need to be closed .. _tutorial_strings: From d23683d21728d9be5a978719fbb75b1cb45b4441 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 16 Feb 2024 08:37:26 +0100 Subject: [PATCH 038/130] Bump numpy from 1.26.1 to 1.26.4 (#1669) Bumps [numpy](https://github.com/numpy/numpy) from 1.26.1 to 1.26.4. - [Release notes](https://github.com/numpy/numpy/releases) - [Changelog](https://github.com/numpy/numpy/blob/main/doc/RELEASE_WALKTHROUGH.rst) - [Commits](https://github.com/numpy/numpy/compare/v1.26.1...v1.26.4) --- updated-dependencies: - dependency-name: numpy dependency-type: direct:development update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- requirements_dev_numpy.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements_dev_numpy.txt b/requirements_dev_numpy.txt index c8c5f7d7ab..d8d6c3d097 100644 --- a/requirements_dev_numpy.txt +++ b/requirements_dev_numpy.txt @@ -1,4 +1,4 @@ # Break this out into a separate file to allow testing against # different versions of numpy. This file should pin to the latest # numpy version. -numpy==1.26.1 +numpy==1.26.4 From 003ff33e70ce0a28411a7e9fde608354b1b8ee9b Mon Sep 17 00:00:00 2001 From: Dimitri Papadopoulos Orfanos <3234522+DimitriPapadopoulos@users.noreply.github.com> Date: Fri, 16 Feb 2024 20:45:43 +0100 Subject: [PATCH 039/130] Change occurrences of % and format() to f-strings (#1423) Co-authored-by: Joe Hamman Co-authored-by: Josh Moore --- docs/release.rst | 3 + zarr/_storage/absstore.py | 6 +- zarr/_storage/store.py | 2 +- zarr/_storage/v3.py | 2 +- zarr/convenience.py | 44 +++++------ zarr/core.py | 20 ++--- zarr/creation.py | 4 +- zarr/errors.py | 4 +- zarr/hierarchy.py | 14 ++-- zarr/indexing.py | 37 +++++----- zarr/meta.py | 10 +-- zarr/meta_v1.py | 4 +- zarr/n5.py | 6 +- zarr/storage.py | 10 +-- zarr/tests/test_core.py | 10 +-- zarr/tests/test_meta.py | 146 ++++++++++++++----------------------- zarr/tests/test_storage.py | 6 +- zarr/util.py | 59 +++++++-------- 18 files changed, 167 insertions(+), 220 deletions(-) diff --git a/docs/release.rst b/docs/release.rst index b73dcec34f..8ce4b2e33c 100644 --- a/docs/release.rst +++ b/docs/release.rst @@ -18,6 +18,9 @@ Release notes Unreleased ---------- +* Change occurrences of % and format() to f-strings. + By :user:`Dimitri Papadopoulos Orfanos ` :issue:`1423`. + .. _release_2.17.0: 2.17.0 diff --git a/zarr/_storage/absstore.py b/zarr/_storage/absstore.py index c9a113148c..b6b386f468 100644 --- a/zarr/_storage/absstore.py +++ b/zarr/_storage/absstore.py @@ -84,7 +84,7 @@ def __init__( blob_service_kwargs = blob_service_kwargs or {} client = ContainerClient( - "https://{}.blob.core.windows.net/".format(account_name), + f"https://{account_name}.blob.core.windows.net/", container, credential=account_key, **blob_service_kwargs, @@ -141,7 +141,7 @@ def __getitem__(self, key): try: return self.client.download_blob(blob_name).readall() except ResourceNotFoundError: - raise KeyError("Blob %s not found" % blob_name) + raise KeyError(f"Blob {blob_name} not found") def __setitem__(self, key, value): value = ensure_bytes(value) @@ -154,7 +154,7 @@ def __delitem__(self, key): try: self.client.delete_blob(self._append_path_to_prefix(key)) except ResourceNotFoundError: - raise KeyError("Blob %s not found" % key) + raise KeyError(f"Blob {key} not found") def __eq__(self, other): return ( diff --git a/zarr/_storage/store.py b/zarr/_storage/store.py index 36b596769a..209f118534 100644 --- a/zarr/_storage/store.py +++ b/zarr/_storage/store.py @@ -227,7 +227,7 @@ def _validate_key(self, key: str): # TODO: Possibly allow key == ".zmetadata" too if we write a # consolidated metadata spec corresponding to this? ): - raise ValueError("keys starts with unexpected value: `{}`".format(key)) + raise ValueError(f"key starts with unexpected value: `{key}`") if key.endswith("/"): raise ValueError("keys may not end in /") diff --git a/zarr/_storage/v3.py b/zarr/_storage/v3.py index 32e78f7a34..56bae74361 100644 --- a/zarr/_storage/v3.py +++ b/zarr/_storage/v3.py @@ -569,7 +569,7 @@ def __init__(self, store: StoreLike, metadata_key=meta_root + "consolidated/.zme consolidated_format = meta.get("zarr_consolidated_format", None) if consolidated_format != 1: raise MetadataError( - "unsupported zarr consolidated metadata format: %s" % consolidated_format + f"unsupported zarr consolidated metadata format: {consolidated_format}" ) # decode metadata diff --git a/zarr/convenience.py b/zarr/convenience.py index b4b8bb5293..7ca5d426f0 100644 --- a/zarr/convenience.py +++ b/zarr/convenience.py @@ -259,7 +259,7 @@ def save_group(store: StoreLike, *args, zarr_version=None, path=None, **kwargs): try: grp = _create_group(_store, path=path, overwrite=True, zarr_version=zarr_version) for i, arr in enumerate(args): - k = "arr_{}".format(i) + k = f"arr_{i}" grp.create_dataset(k, data=arr, overwrite=True, zarr_version=zarr_version) for k, arr in kwargs.items(): grp.create_dataset(k, data=arr, overwrite=True, zarr_version=zarr_version) @@ -499,7 +499,7 @@ def __init__(self, log): self.log_file = log else: raise TypeError( - "log must be a callable function, file path or " "file-like object, found %r" % log + f"log must be a callable function, file path or file-like object, found {log!r}" ) def __enter__(self): @@ -526,9 +526,9 @@ def _log_copy_summary(log, dry_run, n_copied, n_skipped, n_bytes_copied): message = "dry run: " else: message = "all done: " - message += "{:,} copied, {:,} skipped".format(n_copied, n_skipped) + message += f"{n_copied:,} copied, {n_skipped:,} skipped" if not dry_run: - message += ", {:,} bytes copied".format(n_bytes_copied) + message += f", {n_bytes_copied:,} bytes copied" log(message) @@ -657,9 +657,7 @@ def copy_store( # check if_exists parameter valid_if_exists = ["raise", "replace", "skip"] if if_exists not in valid_if_exists: - raise ValueError( - "if_exists must be one of {!r}; found {!r}".format(valid_if_exists, if_exists) - ) + raise ValueError(f"if_exists must be one of {valid_if_exists!r}; found {if_exists!r}") # setup counting variables n_copied = n_skipped = n_bytes_copied = 0 @@ -720,20 +718,20 @@ def copy_store( if if_exists != "replace": if dest_key in dest: if if_exists == "raise": - raise CopyError("key {!r} exists in destination".format(dest_key)) + raise CopyError(f"key {dest_key!r} exists in destination") elif if_exists == "skip": do_copy = False # take action if do_copy: - log("copy {}".format(descr)) + log(f"copy {descr}") if not dry_run: data = source[source_key] n_bytes_copied += buffer_size(data) dest[dest_key] = data n_copied += 1 else: - log("skip {}".format(descr)) + log(f"skip {descr}") n_skipped += 1 # log a final message with a summary of what happened @@ -744,7 +742,7 @@ def copy_store( def _check_dest_is_group(dest): if not hasattr(dest, "create_dataset"): - raise ValueError("dest must be a group, got {!r}".format(dest)) + raise ValueError(f"dest must be a group, got {dest!r}") def copy( @@ -910,11 +908,9 @@ def _copy(log, source, dest, name, root, shallow, without_attrs, if_exists, dry_ # check if_exists parameter valid_if_exists = ["raise", "replace", "skip", "skip_initialized"] if if_exists not in valid_if_exists: - raise ValueError( - "if_exists must be one of {!r}; found {!r}".format(valid_if_exists, if_exists) - ) + raise ValueError(f"if_exists must be one of {valid_if_exists!r}; found {if_exists!r}") if dest_h5py and if_exists == "skip_initialized": - raise ValueError("{!r} can only be used when copying to zarr".format(if_exists)) + raise ValueError(f"{if_exists!r} can only be used when copying to zarr") # determine name to copy to if name is None: @@ -934,9 +930,7 @@ def _copy(log, source, dest, name, root, shallow, without_attrs, if_exists, dry_ exists = dest is not None and name in dest if exists: if if_exists == "raise": - raise CopyError( - "an object {!r} already exists in destination " "{!r}".format(name, dest.name) - ) + raise CopyError(f"an object {name!r} already exists in destination {dest.name!r}") elif if_exists == "skip": do_copy = False elif if_exists == "skip_initialized": @@ -947,7 +941,7 @@ def _copy(log, source, dest, name, root, shallow, without_attrs, if_exists, dry_ # take action if do_copy: # log a message about what we're going to do - log("copy {} {} {}".format(source.name, source.shape, source.dtype)) + log(f"copy {source.name} {source.shape} {source.dtype}") if not dry_run: # clear the way @@ -1015,7 +1009,7 @@ def _copy(log, source, dest, name, root, shallow, without_attrs, if_exists, dry_ n_copied += 1 else: - log("skip {} {} {}".format(source.name, source.shape, source.dtype)) + log(f"skip {source.name} {source.shape} {source.dtype}") n_skipped += 1 elif root or not shallow: @@ -1026,16 +1020,14 @@ def _copy(log, source, dest, name, root, shallow, without_attrs, if_exists, dry_ exists_array = dest is not None and name in dest and hasattr(dest[name], "shape") if exists_array: if if_exists == "raise": - raise CopyError( - "an array {!r} already exists in destination " "{!r}".format(name, dest.name) - ) + raise CopyError(f"an array {name!r} already exists in destination {dest.name!r}") elif if_exists == "skip": do_copy = False # take action if do_copy: # log action - log("copy {}".format(source.name)) + log(f"copy {source.name}") if not dry_run: # clear the way @@ -1078,7 +1070,7 @@ def _copy(log, source, dest, name, root, shallow, without_attrs, if_exists, dry_ n_copied += 1 else: - log("skip {}".format(source.name)) + log(f"skip {source.name}") n_skipped += 1 return n_copied, n_skipped, n_bytes_copied @@ -1327,7 +1319,7 @@ def open_consolidated(store: StoreLike, metadata_key=".zmetadata", mode="r+", ** store, storage_options=kwargs.get("storage_options"), mode=mode, zarr_version=zarr_version ) if mode not in {"r", "r+"}: - raise ValueError("invalid mode, expected either 'r' or 'r+'; found {!r}".format(mode)) + raise ValueError(f"invalid mode, expected either 'r' or 'r+'; found {mode!r}") path = kwargs.pop("path", None) if store._store_version == 2: diff --git a/zarr/core.py b/zarr/core.py index 5727afa884..c3184c6652 100644 --- a/zarr/core.py +++ b/zarr/core.py @@ -2396,11 +2396,11 @@ def _encode_chunk(self, chunk): def __repr__(self): t = type(self) - r = "<{}.{}".format(t.__module__, t.__name__) + r = f"<{t.__module__}.{t.__name__}" if self.name: - r += " %r" % self.name - r += " %s" % str(self.shape) - r += " %s" % self.dtype + r += f" {self.name!r}" + r += f" {str(self.shape)}" + r += f" {self.dtype}" if self._read_only: r += " read-only" r += ">" @@ -2436,11 +2436,11 @@ def info_items(self): def _info_items_nosync(self): def typestr(o): - return "{}.{}".format(type(o).__module__, type(o).__name__) + return f"{type(o).__module__}.{type(o).__name__}" def bytestr(n): if n > 2**10: - return "{} ({})".format(n, human_readable_size(n)) + return f"{n} ({human_readable_size(n)})" else: return str(n) @@ -2451,7 +2451,7 @@ def bytestr(n): items += [("Name", self.name)] items += [ ("Type", typestr(self)), - ("Data type", "%s" % self.dtype), + ("Data type", str(self.dtype)), ("Shape", str(self.shape)), ("Chunk shape", str(self.chunks)), ("Order", self.order), @@ -2461,7 +2461,7 @@ def bytestr(n): # filters if self.filters: for i, f in enumerate(self.filters): - items += [("Filter [%s]" % i, repr(f))] + items += [(f"Filter [{i}]", repr(f))] # compressor items += [("Compressor", repr(self.compressor))] @@ -2478,9 +2478,9 @@ def bytestr(n): if self.nbytes_stored > 0: items += [ ("No. bytes stored", bytestr(self.nbytes_stored)), - ("Storage ratio", "%.1f" % (self.nbytes / self.nbytes_stored)), + ("Storage ratio", f"{self.nbytes / self.nbytes_stored:.1f}"), ] - items += [("Chunks initialized", "{}/{}".format(self.nchunks_initialized, self.nchunks))] + items += [("Chunks initialized", f"{self.nchunks_initialized}/{self.nchunks}")] return items diff --git a/zarr/creation.py b/zarr/creation.py index d4f570895a..264715b040 100644 --- a/zarr/creation.py +++ b/zarr/creation.py @@ -287,7 +287,7 @@ def _kwargs_compat(compressor, fill_value, kwargs): compressor = compression else: - raise ValueError("bad value for compression: %r" % compression) + raise ValueError(f"bad value for compression: {compression!r}") # handle 'fillvalue' if "fillvalue" in kwargs: @@ -297,7 +297,7 @@ def _kwargs_compat(compressor, fill_value, kwargs): # ignore other keyword arguments for k in kwargs: - warn("ignoring keyword argument %r" % k) + warn(f"ignoring keyword argument {k!r}") return compressor, fill_value diff --git a/zarr/errors.py b/zarr/errors.py index 30c9b13d39..85789fbcbf 100644 --- a/zarr/errors.py +++ b/zarr/errors.py @@ -67,9 +67,7 @@ def __init__(self): def err_too_many_indices(selection, shape): - raise IndexError( - "too many indices for array; expected {}, got {}".format(len(shape), len(selection)) - ) + raise IndexError(f"too many indices for array; expected {len(shape)}, got {len(selection)}") class VindexInvalidSelectionError(_BaseZarrIndexError): diff --git a/zarr/hierarchy.py b/zarr/hierarchy.py index 1cfea89c81..44af1d63d1 100644 --- a/zarr/hierarchy.py +++ b/zarr/hierarchy.py @@ -340,9 +340,9 @@ def __len__(self): def __repr__(self): t = type(self) - r = "<{}.{}".format(t.__module__, t.__name__) + r = f"<{t.__module__}.{t.__name__}" if self.name: - r += " %r" % self.name + r += f" {self.name!r}" if self._read_only: r += " read-only" r += ">" @@ -358,7 +358,7 @@ def __exit__(self, exc_type, exc_val, exc_tb): def info_items(self): def typestr(o): - return "{}.{}".format(type(o).__module__, type(o).__name__) + return f"{type(o).__module__}.{type(o).__name__}" items = [] @@ -1157,17 +1157,15 @@ def _require_dataset_nosync(self, name, shape, dtype=None, exact=False, **kwargs shape = normalize_shape(shape) if shape != a.shape: raise TypeError( - "shape do not match existing array; expected {}, got {}".format(a.shape, shape) + f"shape do not match existing array; expected {a.shape}, got {shape}" ) dtype = np.dtype(dtype) if exact: if dtype != a.dtype: - raise TypeError( - "dtypes do not match exactly; expected {}, got {}".format(a.dtype, dtype) - ) + raise TypeError(f"dtypes do not match exactly; expected {a.dtype}, got {dtype}") else: if not np.can_cast(dtype, a.dtype): - raise TypeError("dtypes ({}, {}) cannot be safely cast".format(dtype, a.dtype)) + raise TypeError(f"dtypes ({dtype}, {a.dtype}) cannot be safely cast") return a else: diff --git a/zarr/indexing.py b/zarr/indexing.py index 5a2b7c0eb4..9889fcadad 100644 --- a/zarr/indexing.py +++ b/zarr/indexing.py @@ -338,8 +338,8 @@ def __init__(self, selection, array): else: raise IndexError( - "unsupported selection item for basic indexing; " - "expected integer or slice, got {!r}".format(type(dim_sel)) + f"unsupported selection item for basic indexing; " + f"expected integer or slice, got {type(dim_sel)!r}" ) dim_indexers.append(dim_indexer) @@ -370,8 +370,8 @@ def __init__(self, dim_sel, dim_len, dim_chunk_len): # check shape if dim_sel.shape[0] != dim_len: raise IndexError( - "Boolean array has the wrong length for dimension; " - "expected {}, got {}".format(dim_len, dim_sel.shape[0]) + f"Boolean array has the wrong length for dimension; " + f"expected {dim_len}, got { dim_sel.shape[0]}" ) # store attributes @@ -610,9 +610,9 @@ def __init__(self, selection, array): else: raise IndexError( - "unsupported selection item for orthogonal indexing; " - "expected integer, slice, integer array or Boolean " - "array, got {!r}".format(type(dim_sel)) + f"unsupported selection item for orthogonal indexing; " + f"expected integer, slice, integer array or Boolean " + f"array, got {type(dim_sel)!r}" ) dim_indexers.append(dim_indexer) @@ -698,8 +698,8 @@ def __init__(self, selection, array): if dim_sel.step not in {1, None}: raise IndexError( - "unsupported selection item for block indexing; " - "expected integer or slice with step=1, got {!r}".format(type(dim_sel)) + f"unsupported selection item for block indexing; " + f"expected integer or slice with step=1, got {type(dim_sel)!r}" ) # Can't reuse wraparound_indices because it expects a numpy array @@ -715,8 +715,8 @@ def __init__(self, selection, array): else: raise IndexError( - "unsupported selection item for block indexing; " - "expected integer or slice, got {!r}".format(type(dim_sel)) + f"unsupported selection item for block indexing; " + f"expected integer or slice, got {type(dim_sel)!r}" ) dim_indexer = SliceDimIndexer(slice_, dim_len, dim_chunk_size) @@ -782,9 +782,9 @@ def __init__(self, selection, array): # validation if not is_coordinate_selection(selection, array): raise IndexError( - "invalid coordinate selection; expected one integer " - "(coordinate) array per dimension of the target array, " - "got {!r}".format(selection) + f"invalid coordinate selection; expected one integer " + f"(coordinate) array per dimension of the target array, " + f"got {selection!r}" ) # handle wraparound, boundscheck @@ -874,8 +874,8 @@ def __init__(self, selection, array): # validation if not is_mask_selection(selection, array): raise IndexError( - "invalid mask selection; expected one Boolean (mask)" - "array with the same shape as the target array, got {!r}".format(selection) + f"invalid mask selection; expected one Boolean (mask)" + f"array with the same shape as the target array, got {selection!r}" ) # convert to indices @@ -919,8 +919,7 @@ def check_fields(fields, dtype): # check type if not isinstance(fields, (str, list, tuple)): raise IndexError( - "'fields' argument must be a string or list of strings; found " - "{!r}".format(type(fields)) + f"'fields' argument must be a string or list of strings; found " f"{type(fields)!r}" ) if fields: if dtype.names is None: @@ -933,7 +932,7 @@ def check_fields(fields, dtype): # multiple field selection out_dtype = np.dtype([(f, dtype[f]) for f in fields]) except KeyError as e: - raise IndexError("invalid 'fields' argument, field not found: {!r}".format(e)) + raise IndexError(f"invalid 'fields' argument, field not found: {e!r}") else: return out_dtype else: diff --git a/zarr/meta.py b/zarr/meta.py index d9797e4754..4b360270de 100644 --- a/zarr/meta.py +++ b/zarr/meta.py @@ -111,7 +111,7 @@ def decode_array_metadata(cls, s: Union[MappingType, bytes, str]) -> MappingType # check metadata format zarr_format = meta.get("zarr_format", None) if zarr_format != cls.ZARR_FORMAT: - raise MetadataError("unsupported zarr format: %s" % zarr_format) + raise MetadataError(f"unsupported zarr format: {zarr_format}") # extract array metadata fields try: @@ -199,7 +199,7 @@ def decode_group_metadata(cls, s: Union[MappingType, bytes, str]) -> MappingType # check metadata format version zarr_format = meta.get("zarr_format", None) if zarr_format != cls.ZARR_FORMAT: - raise MetadataError("unsupported zarr format: %s" % zarr_format) + raise MetadataError(f"unsupported zarr format: {zarr_format}") meta = dict(zarr_format=zarr_format) return meta @@ -346,7 +346,7 @@ def decode_group_metadata(cls, s: Union[MappingType, bytes, str]) -> MappingType # # check metadata format version # zarr_format = meta.get("zarr_format", None) # if zarr_format != cls.ZARR_FORMAT: - # raise MetadataError("unsupported zarr format: %s" % zarr_format) + # raise MetadataError(f"unsupported zarr format: {zarr_format}") assert "attributes" in meta # meta = dict(attributes=meta['attributes']) @@ -383,7 +383,7 @@ def decode_hierarchy_metadata(cls, s: Union[MappingType, bytes, str]) -> Mapping # check metadata format # zarr_format = meta.get("zarr_format", None) # if zarr_format != "https://purl.org/zarr/spec/protocol/core/3.0": - # raise MetadataError("unsupported zarr format: %s" % zarr_format) + # raise MetadataError(f"unsupported zarr format: {zarr_format}") if set(meta.keys()) != { "zarr_format", "metadata_encoding", @@ -518,7 +518,7 @@ def decode_array_metadata(cls, s: Union[MappingType, bytes, str]) -> MappingType meta["storage_transformers"] = storage_transformers except Exception as e: - raise MetadataError("error decoding metadata: %s" % e) + raise MetadataError(f"error decoding metadata: {e}") else: return meta diff --git a/zarr/meta_v1.py b/zarr/meta_v1.py index 4ac381f2ca..65bfd3488e 100644 --- a/zarr/meta_v1.py +++ b/zarr/meta_v1.py @@ -10,7 +10,7 @@ def decode_metadata(b): meta = json.loads(s) zarr_format = meta.get("zarr_format", None) if zarr_format != 1: - raise MetadataError("unsupported zarr format: %s" % zarr_format) + raise MetadataError(f"unsupported zarr format: {zarr_format}") try: meta = dict( zarr_format=meta["zarr_format"], @@ -23,7 +23,7 @@ def decode_metadata(b): order=meta["order"], ) except Exception as e: - raise MetadataError("error decoding metadata: %s" % e) + raise MetadataError(f"error decoding metadata: {e}") else: return meta diff --git a/zarr/n5.py b/zarr/n5.py index c50c18f718..fdd3d5babf 100644 --- a/zarr/n5.py +++ b/zarr/n5.py @@ -826,9 +826,9 @@ def decode(self, chunk, out=None) -> bytes: if out is not None: # out should only be used if we read a complete chunk - assert chunk_shape == self.chunk_shape, "Expected chunk of shape {}, found {}".format( - self.chunk_shape, chunk_shape - ) + assert ( + chunk_shape == self.chunk_shape + ), f"Expected chunk of shape {self.chunk_shape}, found {chunk_shape}" if self._compressor: self._compressor.decode(chunk, out) diff --git a/zarr/storage.py b/zarr/storage.py index a26dc636db..73a6dc9630 100644 --- a/zarr/storage.py +++ b/zarr/storage.py @@ -2700,14 +2700,12 @@ def listdir(self, path=None): path = normalize_storage_path(path) sep = "_" if path == "" else "/" keys = self.cursor.execute( - """ + f""" SELECT DISTINCT SUBSTR(m, 0, INSTR(m, "/")) AS l FROM ( SELECT LTRIM(SUBSTR(k, LENGTH(?) + 1), "/") || "/" AS m FROM zarr WHERE k LIKE (? || "{sep}%") ) ORDER BY l ASC - """.format( - sep=sep - ), + """, (path, path), ) keys = list(map(operator.itemgetter(0), keys)) @@ -2863,7 +2861,7 @@ def __init__(self, prefix="zarr", dimension_separator=None, **kwargs): self.client = redis.Redis(**kwargs) def _key(self, key): - return "{prefix}:{key}".format(prefix=self._prefix, key=key) + return f"{self._prefix}:{key}" def __getitem__(self, key): return self.client[self._key(key)] @@ -2948,7 +2946,7 @@ def __init__(self, store: StoreLike, metadata_key=".zmetadata"): consolidated_format = meta.get("zarr_consolidated_format", None) if consolidated_format != 1: raise MetadataError( - "unsupported zarr consolidated metadata format: %s" % consolidated_format + f"unsupported zarr consolidated metadata format: {consolidated_format}" ) # decode metadata diff --git a/zarr/tests/test_core.py b/zarr/tests/test_core.py index cf15703497..d9447c0832 100644 --- a/zarr/tests/test_core.py +++ b/zarr/tests/test_core.py @@ -188,7 +188,7 @@ def test_store_has_text_keys(self): for k in z.chunk_store.keys(): if not isinstance(k, expected_type): # pragma: no cover - pytest.fail("Non-text key: %s" % repr(k)) + pytest.fail(f"Non-text key: {k!r}") z.store.close() @@ -202,7 +202,7 @@ def test_store_has_binary_values(self): try: ensure_ndarray(v) except TypeError: # pragma: no cover - pytest.fail("Non-bytes-like value: %s" % repr(v)) + pytest.fail(f"Non-bytes-like value: {v!r}") z.store.close() @@ -1212,7 +1212,7 @@ def test_dtypes(self): # datetime, timedelta for base_type in "Mm": for resolution in "D", "us", "ns": - dtype = "{}8[{}]".format(base_type, resolution) + dtype = f"{base_type}8[{resolution}]" z = self.create_array(shape=100, dtype=dtype, fill_value=0) assert z.dtype == np.dtype(dtype) a = np.random.randint( @@ -1402,7 +1402,7 @@ def compare_arrays(expected, actual, item_dtype): # convenience API for item_type in "int", " Tuple[np.dtype object_codec = codec_registry[codec_id](*args) except KeyError: # pragma: no cover raise ValueError( - "codec %r for object type %r is not " - "available; please provide an " - "object_codec manually" % (codec_id, key) + f"codec {codec_id!r} for object type {key!r} is not " + f"available; please provide an object_codec manually" ) return dtype, object_codec @@ -241,7 +240,7 @@ def is_total_slice(item, shape: Tuple[int]) -> bool: for it, sh in zip(item, shape) ) else: - raise TypeError("expected slice or tuple of slices, found %r" % item) + raise TypeError(f"expected slice or tuple of slices, found {item!r}") def normalize_resize_args(old_shape, *args): @@ -265,23 +264,23 @@ def normalize_resize_args(old_shape, *args): def human_readable_size(size) -> str: if size < 2**10: - return "%s" % size + return f"{size}" elif size < 2**20: - return "%.1fK" % (size / float(2**10)) + return f"{size / float(2**10):.1f}K" elif size < 2**30: - return "%.1fM" % (size / float(2**20)) + return f"{size / float(2**20):.1f}M" elif size < 2**40: - return "%.1fG" % (size / float(2**30)) + return f"{size / float(2**30):.1f}G" elif size < 2**50: - return "%.1fT" % (size / float(2**40)) + return f"{size / float(2**40):.1f}T" else: - return "%.1fP" % (size / float(2**50)) + return f"{size / float(2**50):.1f}P" def normalize_order(order: str) -> str: order = str(order).upper() if order not in ["C", "F"]: - raise ValueError("order must be either 'C' or 'F', found: %r" % order) + raise ValueError(f"order must be either 'C' or 'F', found: {order!r}") return order @@ -289,7 +288,7 @@ def normalize_dimension_separator(sep: Optional[str]) -> Optional[DIMENSION_SEPA if sep in (".", "/", None): return cast(Optional[DIMENSION_SEPARATOR], sep) else: - raise ValueError("dimension_separator must be either '.' or '/', found: %r" % sep) + raise ValueError(f"dimension_separator must be either '.' or '/', found: {sep!r}") def normalize_fill_value(fill_value, dtype: np.dtype): @@ -307,8 +306,8 @@ def normalize_fill_value(fill_value, dtype: np.dtype): if not isinstance(fill_value, str): raise ValueError( - "fill_value {!r} is not valid for dtype {}; must be a " - "unicode string".format(fill_value, dtype) + f"fill_value {fill_value!r} is not valid for dtype {dtype}; " + f"must be a unicode string" ) else: @@ -322,8 +321,8 @@ def normalize_fill_value(fill_value, dtype: np.dtype): except Exception as e: # re-raise with our own error message to be helpful raise ValueError( - "fill_value {!r} is not valid for dtype {}; nested " - "exception: {}".format(fill_value, dtype, e) + f"fill_value {fill_value!r} is not valid for dtype {dtype}; " + f"nested exception: {e}" ) return fill_value @@ -396,10 +395,10 @@ def info_html_report(items) -> str: report += "" for k, v in items: report += ( - "" - '%s' - '%s' - "" % (k, v) + f"" + f'{k}' + f'{v}' + f"" ) report += "" report += "" @@ -435,7 +434,7 @@ def get_children(self): def get_text(self): name = self.obj.name.split("/")[-1] or "/" if hasattr(self.obj, "shape"): - name += " {} {}".format(self.obj.shape, self.obj.dtype) + name += f" {self.obj.shape} {self.obj.dtype}" return name def get_type(self): @@ -463,7 +462,7 @@ def tree_get_icon(stype: str) -> str: elif stype == "Group": return tree_group_icon else: - raise ValueError("Unknown type: %s" % stype) + raise ValueError(f"Unknown type: {stype}") def tree_widget_sublist(node, root=False, expand=False): @@ -487,10 +486,10 @@ def tree_widget(group, expand, level): import ipytree except ImportError as error: raise ImportError( - "{}: Run `pip install zarr[jupyter]` or `conda install ipytree`" - "to get the required ipytree dependency for displaying the tree " - "widget. If using jupyterlab<3, you also need to run " - "`jupyter labextension install ipytree`".format(error) + f"{error}: Run `pip install zarr[jupyter]` or `conda install ipytree`" + f"to get the required ipytree dependency for displaying the tree " + f"widget. If using jupyterlab<3, you also need to run " + f"`jupyter labextension install ipytree`" ) result = ipytree.Tree() @@ -549,14 +548,10 @@ def _repr_mimebundle_(self, **kwargs): def check_array_shape(param, array, shape): if not hasattr(array, "shape"): - raise TypeError( - "parameter {!r}: expected an array-like object, got {!r}".format(param, type(array)) - ) + raise TypeError(f"parameter {param!r}: expected an array-like object, got {type(array)!r}") if array.shape != shape: raise ValueError( - "parameter {!r}: expected array with shape {!r}, got {!r}".format( - param, shape, array.shape - ) + f"parameter {param!r}: expected array with shape {shape!r}, got {array.shape!r}" ) From f80f697c2612cf41c5bdb158a602c1ae8a737e70 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 20 Feb 2024 18:28:10 +0530 Subject: [PATCH 040/130] chore: update pre-commit hooks (#1672) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit updates: - [github.com/astral-sh/ruff-pre-commit: v0.2.1 → v0.2.2](https://github.com/astral-sh/ruff-pre-commit/compare/v0.2.1...v0.2.2) Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index c7d4f32c68..41b65f1d02 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -8,7 +8,7 @@ default_language_version: repos: - repo: https://github.com/astral-sh/ruff-pre-commit # Ruff version. - rev: 'v0.2.1' + rev: 'v0.2.2' hooks: - id: ruff - repo: https://github.com/psf/black From 54bc90c8682472cc40fba35ec6b313cb1f046c34 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 26 Feb 2024 09:57:06 -0800 Subject: [PATCH 041/130] Bump pymongo from 4.6.1 to 4.6.2 (#1674) Bumps [pymongo](https://github.com/mongodb/mongo-python-driver) from 4.6.1 to 4.6.2. - [Release notes](https://github.com/mongodb/mongo-python-driver/releases) - [Changelog](https://github.com/mongodb/mongo-python-driver/blob/4.6.2/doc/changelog.rst) - [Commits](https://github.com/mongodb/mongo-python-driver/compare/4.6.1...4.6.2) --- updated-dependencies: - dependency-name: pymongo dependency-type: direct:development update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- requirements_dev_optional.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements_dev_optional.txt b/requirements_dev_optional.txt index e94b814173..85f6fccffc 100644 --- a/requirements_dev_optional.txt +++ b/requirements_dev_optional.txt @@ -11,7 +11,7 @@ azure-storage-blob==12.16.0 # pyup: ignore redis==5.0.1 types-redis types-setuptools -pymongo==4.6.1 +pymongo==4.6.2 # optional test requirements coverage pytest-cov==4.1.0 From 70a15bbe595031ad24b82ca5cee9468a8229e775 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 26 Feb 2024 10:09:11 -0800 Subject: [PATCH 042/130] Bump conda-incubator/setup-miniconda from 3.0.1 to 3.0.2 (#1677) Bumps [conda-incubator/setup-miniconda](https://github.com/conda-incubator/setup-miniconda) from 3.0.1 to 3.0.2. - [Release notes](https://github.com/conda-incubator/setup-miniconda/releases) - [Changelog](https://github.com/conda-incubator/setup-miniconda/blob/main/CHANGELOG.md) - [Commits](https://github.com/conda-incubator/setup-miniconda/compare/v3.0.1...v3.0.2) --- updated-dependencies: - dependency-name: conda-incubator/setup-miniconda dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/minimal.yml | 2 +- .github/workflows/python-package.yml | 2 +- .github/workflows/windows-testing.yml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/minimal.yml b/.github/workflows/minimal.yml index 2cc0213781..d95b2bc540 100644 --- a/.github/workflows/minimal.yml +++ b/.github/workflows/minimal.yml @@ -15,7 +15,7 @@ jobs: steps: - uses: actions/checkout@v4 - name: Setup Miniconda - uses: conda-incubator/setup-miniconda@v3.0.1 + uses: conda-incubator/setup-miniconda@v3.0.2 with: channels: conda-forge environment-file: environment.yml diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index d74df9ce67..946b7efa7d 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -42,7 +42,7 @@ jobs: with: fetch-depth: 0 - name: Setup Miniconda - uses: conda-incubator/setup-miniconda@v3.0.1 + uses: conda-incubator/setup-miniconda@v3.0.2 with: channels: conda-forge python-version: ${{ matrix.python-version }} diff --git a/.github/workflows/windows-testing.yml b/.github/workflows/windows-testing.yml index 0ef7f21758..85e5c3e6b6 100644 --- a/.github/workflows/windows-testing.yml +++ b/.github/workflows/windows-testing.yml @@ -21,7 +21,7 @@ jobs: - uses: actions/checkout@v4 with: fetch-depth: 0 - - uses: conda-incubator/setup-miniconda@v3.0.1 + - uses: conda-incubator/setup-miniconda@v3.0.2 with: auto-update-conda: true python-version: ${{ matrix.python-version }} From ec4d2162828c2616a388dda2bdbcf40c8747a36d Mon Sep 17 00:00:00 2001 From: Josh Moore Date: Tue, 27 Feb 2024 15:26:29 +0100 Subject: [PATCH 043/130] Update config.yml with Zulip --- .github/ISSUE_TEMPLATE/config.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml index 9cb5ec9a78..907121f858 100644 --- a/.github/ISSUE_TEMPLATE/config.yml +++ b/.github/ISSUE_TEMPLATE/config.yml @@ -3,8 +3,8 @@ contact_links: - name: ✨ Propose a new major feature url: https://github.com/zarr-developers/zarr-specs about: A new major feature should be discussed in the Zarr specifications repository. - - name: ❓ Discuss something on gitter - url: https://gitter.im/zarr-developers/community + - name: ❓ Discuss something on Zulip + url: https://ossci.zulipchat.com/ about: For questions like "How do I do X with Zarr?", you can move to our Gitter channel. - name: ❓ Discuss something on GitHub Discussions url: https://github.com/zarr-developers/zarr-python/discussions From a0e5559c38bf1a9d7c1a70a81f51f5eece5701c2 Mon Sep 17 00:00:00 2001 From: David Stansby Date: Tue, 27 Feb 2024 17:04:50 +0100 Subject: [PATCH 044/130] Type dimension separator (#1620) Co-authored-by: Davis Bennett --- zarr/_storage/absstore.py | 5 ++- zarr/_storage/v3.py | 7 ++-- zarr/_storage/v3_storage_transformers.py | 3 +- zarr/creation.py | 2 +- zarr/storage.py | 45 +++++++++++++++++------- 5 files changed, 44 insertions(+), 18 deletions(-) diff --git a/zarr/_storage/absstore.py b/zarr/_storage/absstore.py index b6b386f468..217b2a29e0 100644 --- a/zarr/_storage/absstore.py +++ b/zarr/_storage/absstore.py @@ -1,9 +1,12 @@ """This module contains storage classes related to Azure Blob Storage (ABS)""" +from typing import Optional import warnings + from numcodecs.compat import ensure_bytes from zarr.util import normalize_storage_path from zarr._storage.store import _get_metadata_suffix, data_root, meta_root, Store, StoreV3 +from zarr.types import DIMENSION_SEPARATOR __doctest_requires__ = { ("ABSStore", "ABSStore.*"): ["azure.storage.blob"], @@ -67,7 +70,7 @@ def __init__( account_name=None, account_key=None, blob_service_kwargs=None, - dimension_separator=None, + dimension_separator: Optional[DIMENSION_SEPARATOR] = None, client=None, ): self._dimension_separator = dimension_separator diff --git a/zarr/_storage/v3.py b/zarr/_storage/v3.py index 56bae74361..4987f820cf 100644 --- a/zarr/_storage/v3.py +++ b/zarr/_storage/v3.py @@ -3,13 +3,14 @@ from collections import OrderedDict from collections.abc import MutableMapping from threading import Lock -from typing import Union, Dict, Any +from typing import Union, Dict, Any, Optional from zarr.errors import ( MetadataError, ReadOnlyError, ) from zarr.util import buffer_size, json_loads, normalize_storage_path +from zarr.types import DIMENSION_SEPARATOR from zarr._storage.absstore import ABSStoreV3 # noqa: F401 from zarr._storage.store import ( # noqa: F401 @@ -224,7 +225,9 @@ def get_partial_values(self, key_ranges): class MemoryStoreV3(MemoryStore, StoreV3): - def __init__(self, root=None, cls=dict, dimension_separator=None): + def __init__( + self, root=None, cls=dict, dimension_separator: Optional[DIMENSION_SEPARATOR] = None + ): if root is None: self.root = cls() else: diff --git a/zarr/_storage/v3_storage_transformers.py b/zarr/_storage/v3_storage_transformers.py index 3afc3823a3..37e56f8ecd 100644 --- a/zarr/_storage/v3_storage_transformers.py +++ b/zarr/_storage/v3_storage_transformers.py @@ -8,6 +8,7 @@ from zarr._storage.store import StorageTransformer, StoreV3, _rmdir_from_keys_v3 from zarr.util import normalize_storage_path +from zarr.types import DIMENSION_SEPARATOR MAX_UINT_64 = 2**64 - 1 @@ -118,7 +119,7 @@ def _copy_for_array(self, array, inner_store): return transformer_copy @property - def dimension_separator(self) -> str: + def dimension_separator(self) -> DIMENSION_SEPARATOR: assert ( self._dimension_separator is not None ), "dimension_separator is not initialized, first get a copy via _copy_for_array." diff --git a/zarr/creation.py b/zarr/creation.py index 264715b040..c541531d54 100644 --- a/zarr/creation.py +++ b/zarr/creation.py @@ -470,7 +470,7 @@ def open_array( write_empty_chunks=True, *, zarr_version=None, - dimension_separator=None, + dimension_separator: Optional[DIMENSION_SEPARATOR] = None, meta_array=None, **kwargs, ): diff --git a/zarr/storage.py b/zarr/storage.py index 73a6dc9630..f6903d29b2 100644 --- a/zarr/storage.py +++ b/zarr/storage.py @@ -41,7 +41,8 @@ from numcodecs.compat import ensure_bytes, ensure_text, ensure_contiguous_ndarray_like from numcodecs.registry import codec_registry from zarr.context import Context -from zarr.types import PathLike as Path +from zarr.types import PathLike as Path, DIMENSION_SEPARATOR +from zarr.util import NoLock from zarr.errors import ( MetadataError, @@ -327,7 +328,7 @@ def init_array( chunk_store: Optional[StoreLike] = None, filters=None, object_codec=None, - dimension_separator=None, + dimension_separator: Optional[DIMENSION_SEPARATOR] = None, storage_transformers=(), ): """Initialize an array store with the given configuration. Note that this is a low-level @@ -481,7 +482,7 @@ def _init_array_metadata( chunk_store: Optional[StoreLike] = None, filters=None, object_codec=None, - dimension_separator=None, + dimension_separator: Optional[DIMENSION_SEPARATOR] = None, storage_transformers=(), ): store_version = getattr(store, "_store_version", 2) @@ -1054,7 +1055,9 @@ class DirectoryStore(Store): """ - def __init__(self, path, normalize_keys=False, dimension_separator=None): + def __init__( + self, path, normalize_keys=False, dimension_separator: Optional[DIMENSION_SEPARATOR] = None + ): # guard conditions path = os.path.abspath(path) if os.path.exists(path) and not os.path.isdir(path): @@ -1349,7 +1352,7 @@ def __init__( key_separator=None, mode="w", exceptions=(KeyError, PermissionError, IOError), - dimension_separator=None, + dimension_separator: Optional[DIMENSION_SEPARATOR] = None, fs=None, check=False, create=False, @@ -1568,7 +1571,12 @@ class TempStore(DirectoryStore): # noinspection PyShadowingBuiltins def __init__( - self, suffix="", prefix="zarr", dir=None, normalize_keys=False, dimension_separator=None + self, + suffix="", + prefix="zarr", + dir=None, + normalize_keys=False, + dimension_separator: Optional[DIMENSION_SEPARATOR] = None, ): path = tempfile.mkdtemp(suffix=suffix, prefix=prefix, dir=dir) atexit.register(atexit_rmtree, path) @@ -1652,7 +1660,9 @@ class NestedDirectoryStore(DirectoryStore): """ - def __init__(self, path, normalize_keys=False, dimension_separator="/"): + def __init__( + self, path, normalize_keys=False, dimension_separator: Optional[DIMENSION_SEPARATOR] = "/" + ): super().__init__(path, normalize_keys=normalize_keys) if dimension_separator is None: dimension_separator = "/" @@ -1765,7 +1775,7 @@ def __init__( compression=zipfile.ZIP_STORED, allowZip64=True, mode="a", - dimension_separator=None, + dimension_separator: Optional[DIMENSION_SEPARATOR] = None, ): # store properties path = os.path.abspath(path) @@ -2058,7 +2068,7 @@ def __init__( mode=0o666, open=None, write_lock=True, - dimension_separator=None, + dimension_separator: Optional[DIMENSION_SEPARATOR] = None, **open_kwargs, ): if open is None: @@ -2073,6 +2083,7 @@ def __init__( self.mode = mode self.open = open self.write_lock = write_lock + self.write_mutex: Union[Lock, NoLock] if write_lock: # This may not be required as some dbm implementations manage their own # locks, but err on the side of caution. @@ -2229,7 +2240,13 @@ class LMDBStore(Store): """ - def __init__(self, path, buffers=True, dimension_separator=None, **kwargs): + def __init__( + self, + path, + buffers=True, + dimension_separator: Optional[DIMENSION_SEPARATOR] = None, + **kwargs, + ): import lmdb # set default memory map size to something larger than the lmdb default, which is @@ -2580,7 +2597,7 @@ class SQLiteStore(Store): >>> store.close() # don't forget to call this when you're done """ - def __init__(self, path, dimension_separator=None, **kwargs): + def __init__(self, path, dimension_separator: Optional[DIMENSION_SEPARATOR] = None, **kwargs): import sqlite3 self._dimension_separator = dimension_separator @@ -2776,7 +2793,7 @@ def __init__( self, database="mongodb_zarr", collection="zarr_collection", - dimension_separator=None, + dimension_separator: Optional[DIMENSION_SEPARATOR] = None, **kwargs, ): import pymongo @@ -2851,7 +2868,9 @@ class RedisStore(Store): """ - def __init__(self, prefix="zarr", dimension_separator=None, **kwargs): + def __init__( + self, prefix="zarr", dimension_separator: Optional[DIMENSION_SEPARATOR] = None, **kwargs + ): import redis self._prefix = prefix From 99e03c684729b188457024a53afc45cb1b160027 Mon Sep 17 00:00:00 2001 From: Sanket Verma Date: Wed, 28 Feb 2024 18:22:28 +0530 Subject: [PATCH 045/130] Replace Gitter with new Zulip Chat link (#1685) * Replace Gitter with Zulip * Replace Gitter with Zulip in remaining places --- .github/ISSUE_TEMPLATE/config.yml | 2 +- README.md | 6 +++--- docs/index.rst | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml index 907121f858..9ceaab2ae7 100644 --- a/.github/ISSUE_TEMPLATE/config.yml +++ b/.github/ISSUE_TEMPLATE/config.yml @@ -5,7 +5,7 @@ contact_links: about: A new major feature should be discussed in the Zarr specifications repository. - name: ❓ Discuss something on Zulip url: https://ossci.zulipchat.com/ - about: For questions like "How do I do X with Zarr?", you can move to our Gitter channel. + about: For questions like "How do I do X with Zarr?", you can move to our Zulip Chat. - name: ❓ Discuss something on GitHub Discussions url: https://github.com/zarr-developers/zarr-python/discussions about: For questions like "How do I do X with Zarr?", you can move to GitHub Discussions. diff --git a/README.md b/README.md index b035ffa597..e379c9719f 100644 --- a/README.md +++ b/README.md @@ -70,10 +70,10 @@ - Gitter + Zulip - - + + diff --git a/docs/index.rst b/docs/index.rst index 06f79b7e7c..a5dbfbc5bf 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -25,7 +25,7 @@ Zarr-Python `Installation `_ | `Source Repository `_ | `Issue Tracker `_ | -`Gitter `_ +`Zulip Chat `_ Zarr is a file storage format for chunked, compressed, N-dimensional arrays based on an open-source specification. From 67d5d82317451c9072a187efd6d638e718cdaced Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 2 Mar 2024 02:03:35 +0530 Subject: [PATCH 046/130] Bump redis from 5.0.1 to 5.0.2 (#1688) Bumps [redis](https://github.com/redis/redis-py) from 5.0.1 to 5.0.2. - [Release notes](https://github.com/redis/redis-py/releases) - [Changelog](https://github.com/redis/redis-py/blob/master/CHANGES) - [Commits](https://github.com/redis/redis-py/compare/v5.0.1...v5.0.2) --- updated-dependencies: - dependency-name: redis dependency-type: direct:development update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- requirements_dev_optional.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements_dev_optional.txt b/requirements_dev_optional.txt index 85f6fccffc..c3d747a47e 100644 --- a/requirements_dev_optional.txt +++ b/requirements_dev_optional.txt @@ -8,7 +8,7 @@ ipywidgets==8.1.2 # don't let pyup change pinning for azure-storage-blob, need to pin to older # version to get compatibility with azure storage emulator on appveyor (FIXME) azure-storage-blob==12.16.0 # pyup: ignore -redis==5.0.1 +redis==5.0.2 types-redis types-setuptools pymongo==4.6.2 From 9c2a412d70ed717165966bc47615bdef195d68c5 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 4 Mar 2024 15:48:15 +0100 Subject: [PATCH 047/130] Bump pypa/gh-action-pypi-publish from 1.8.11 to 1.8.12 (#1691) Bumps [pypa/gh-action-pypi-publish](https://github.com/pypa/gh-action-pypi-publish) from 1.8.11 to 1.8.12. - [Release notes](https://github.com/pypa/gh-action-pypi-publish/releases) - [Commits](https://github.com/pypa/gh-action-pypi-publish/compare/v1.8.11...v1.8.12) --- updated-dependencies: - dependency-name: pypa/gh-action-pypi-publish dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/releases.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/releases.yml b/.github/workflows/releases.yml index 250c6112c8..6d417042b5 100644 --- a/.github/workflows/releases.yml +++ b/.github/workflows/releases.yml @@ -64,7 +64,7 @@ jobs: with: name: releases path: dist - - uses: pypa/gh-action-pypi-publish@v1.8.11 + - uses: pypa/gh-action-pypi-publish@v1.8.12 with: user: __token__ password: ${{ secrets.pypi_password }} From 237f934f5ac7d7a04c6b144f97e54776eda628c6 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 6 Mar 2024 01:02:48 +0530 Subject: [PATCH 048/130] Bump pytest-doctestplus from 1.1.0 to 1.2.0 (#1693) Bumps [pytest-doctestplus](https://github.com/scientific-python/pytest-doctestplus) from 1.1.0 to 1.2.0. - [Release notes](https://github.com/scientific-python/pytest-doctestplus/releases) - [Changelog](https://github.com/scientific-python/pytest-doctestplus/blob/main/CHANGES.rst) - [Commits](https://github.com/scientific-python/pytest-doctestplus/compare/v1.1.0...v1.2.0) --- updated-dependencies: - dependency-name: pytest-doctestplus dependency-type: direct:development update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- requirements_dev_optional.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements_dev_optional.txt b/requirements_dev_optional.txt index c3d747a47e..0f4493b1d4 100644 --- a/requirements_dev_optional.txt +++ b/requirements_dev_optional.txt @@ -15,7 +15,7 @@ pymongo==4.6.2 # optional test requirements coverage pytest-cov==4.1.0 -pytest-doctestplus==1.1.0 +pytest-doctestplus==1.2.0 pytest-timeout==2.2.0 h5py==3.10.0 fsspec==2023.12.2 From 240bb824b86a68aa0eb6f03fecbe5439882607ca Mon Sep 17 00:00:00 2001 From: Sanket Verma Date: Wed, 6 Mar 2024 19:48:22 +0530 Subject: [PATCH 049/130] Fix RTD build (#1694) --- .readthedocs.yaml | 4 +++- docs/index.rst | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/.readthedocs.yaml b/.readthedocs.yaml index 08cac8d78d..e45cae1b45 100644 --- a/.readthedocs.yaml +++ b/.readthedocs.yaml @@ -16,4 +16,6 @@ python: extra_requirements: - docs -formats: all +formats: + - htmlzip + - pdf diff --git a/docs/index.rst b/docs/index.rst index a5dbfbc5bf..cf54e261af 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -19,7 +19,7 @@ Zarr-Python **Version**: |version| -**Download documentation**: `PDF/Zipped HTML/EPUB `_ +**Download documentation**: `PDF/Zipped HTML `_ **Useful links**: `Installation `_ | From a1fbedb18c1fc70f026c423fafca6d84ad88ce53 Mon Sep 17 00:00:00 2001 From: Sanket Verma Date: Thu, 7 Mar 2024 00:53:47 +0530 Subject: [PATCH 050/130] Update release.rst for v2.17.1 (#1673) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Update release.rst for v2.17.1 * Change the copyright year from 2023 → 2024. * Update release.rst for v2.17.1 --- LICENSE.txt | 2 +- docs/conf.py | 2 +- docs/release.rst | 32 ++++++++++++++++++++++++++++++++ 3 files changed, 34 insertions(+), 2 deletions(-) diff --git a/LICENSE.txt b/LICENSE.txt index 850a0d8772..a4de1c39d3 100644 --- a/LICENSE.txt +++ b/LICENSE.txt @@ -1,6 +1,6 @@ The MIT License (MIT) -Copyright (c) 2015-2023 Zarr Developers +Copyright (c) 2015-2024 Zarr Developers Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/docs/conf.py b/docs/conf.py index 318843a9fb..048e77f51d 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -72,7 +72,7 @@ # General information about the project. project = "zarr" -copyright = "2023, Zarr Developers" +copyright = "2024, Zarr Developers" author = "Zarr Developers" version = zarr.__version__ diff --git a/docs/release.rst b/docs/release.rst index 8ce4b2e33c..037432ca58 100644 --- a/docs/release.rst +++ b/docs/release.rst @@ -18,9 +18,41 @@ Release notes Unreleased ---------- +.. _release_2.17.1: + +2.17.1 +------ + +Enhancements +~~~~~~~~~~~~ + * Change occurrences of % and format() to f-strings. By :user:`Dimitri Papadopoulos Orfanos ` :issue:`1423`. +* Proper argument for numpy.reshape. + By :user:`Dimitri Papadopoulos Orfanos ` :issue:`1425`. + +* Add typing to dimension separator arguments. + By :user:`David Stansby ` :issue:`1620`. + +Docs +~~~~ + +* ZIP related tweaks. + By :user:`Davis Bennett ` :issue:`1641`. + +Maintenance +~~~~~~~~~~~ + +* Update config.yml with Zulip. + By :user:`Josh Moore `. + +* Replace Gitter with the new Zulip Chat link. + By :user:`Sanket Verma ` :issue:`1685`. + +* Fix RTD build. + By :user:`Sanket Verma ` :issue:`1694`. + .. _release_2.17.0: 2.17.0 From d986f8973eafbf847179d8c3f7e16451d0fcd63d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 8 Mar 2024 17:40:55 -0800 Subject: [PATCH 051/130] Bump pytest-timeout from 2.2.0 to 2.3.1 (#1697) Bumps [pytest-timeout](https://github.com/pytest-dev/pytest-timeout) from 2.2.0 to 2.3.1. - [Commits](https://github.com/pytest-dev/pytest-timeout/compare/2.2.0...2.3.1) --- updated-dependencies: - dependency-name: pytest-timeout dependency-type: direct:development update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- requirements_dev_optional.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements_dev_optional.txt b/requirements_dev_optional.txt index 0f4493b1d4..b14381dd6e 100644 --- a/requirements_dev_optional.txt +++ b/requirements_dev_optional.txt @@ -16,7 +16,7 @@ pymongo==4.6.2 coverage pytest-cov==4.1.0 pytest-doctestplus==1.2.0 -pytest-timeout==2.2.0 +pytest-timeout==2.3.1 h5py==3.10.0 fsspec==2023.12.2 s3fs==2023.12.2 From d642da6320793c64dc227cf8062a9936d0fd398e Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sun, 10 Mar 2024 14:26:24 -0700 Subject: [PATCH 052/130] Bump conda-incubator/setup-miniconda from 3.0.2 to 3.0.3 (#1690) Bumps [conda-incubator/setup-miniconda](https://github.com/conda-incubator/setup-miniconda) from 3.0.2 to 3.0.3. - [Release notes](https://github.com/conda-incubator/setup-miniconda/releases) - [Changelog](https://github.com/conda-incubator/setup-miniconda/blob/main/CHANGELOG.md) - [Commits](https://github.com/conda-incubator/setup-miniconda/compare/v3.0.2...v3.0.3) --- updated-dependencies: - dependency-name: conda-incubator/setup-miniconda dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Sanket Verma --- .github/workflows/minimal.yml | 2 +- .github/workflows/python-package.yml | 2 +- .github/workflows/windows-testing.yml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/minimal.yml b/.github/workflows/minimal.yml index d95b2bc540..dba6918514 100644 --- a/.github/workflows/minimal.yml +++ b/.github/workflows/minimal.yml @@ -15,7 +15,7 @@ jobs: steps: - uses: actions/checkout@v4 - name: Setup Miniconda - uses: conda-incubator/setup-miniconda@v3.0.2 + uses: conda-incubator/setup-miniconda@v3.0.3 with: channels: conda-forge environment-file: environment.yml diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index 946b7efa7d..fd2603ff95 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -42,7 +42,7 @@ jobs: with: fetch-depth: 0 - name: Setup Miniconda - uses: conda-incubator/setup-miniconda@v3.0.2 + uses: conda-incubator/setup-miniconda@v3.0.3 with: channels: conda-forge python-version: ${{ matrix.python-version }} diff --git a/.github/workflows/windows-testing.yml b/.github/workflows/windows-testing.yml index 85e5c3e6b6..d580ef3f0e 100644 --- a/.github/workflows/windows-testing.yml +++ b/.github/workflows/windows-testing.yml @@ -21,7 +21,7 @@ jobs: - uses: actions/checkout@v4 with: fetch-depth: 0 - - uses: conda-incubator/setup-miniconda@v3.0.2 + - uses: conda-incubator/setup-miniconda@v3.0.3 with: auto-update-conda: true python-version: ${{ matrix.python-version }} From 029cff71b86871cde76c7909cfecd28764953377 Mon Sep 17 00:00:00 2001 From: "Daniel Jahn (dahn)" Date: Sun, 10 Mar 2024 22:37:36 +0100 Subject: [PATCH 053/130] docs(tutorial.rst): fix link to GCSMap (#1689) --- docs/tutorial.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/tutorial.rst b/docs/tutorial.rst index 1f7accab3a..214dd4f63f 100644 --- a/docs/tutorial.rst +++ b/docs/tutorial.rst @@ -868,7 +868,7 @@ implementations of the ``MutableMapping`` interface for Amazon S3 (`S3Map Distributed File System (`HDFSMap `_) and Google Cloud Storage (`GCSMap -`_), which +`_), which can be used with Zarr. Here is an example using S3Map to read an array created previously:: From 9fc4981ddfb0e032fcc76fa6585b5a66dc5d2f06 Mon Sep 17 00:00:00 2001 From: Sanket Verma Date: Mon, 11 Mar 2024 03:19:28 +0530 Subject: [PATCH 054/130] Update installation.rst stating version support policy (#1665) * Update installation.rst stating version support policy * Update docs/installation.rst Co-authored-by: Joe Hamman * Update docs/installation.rst --------- Co-authored-by: Joe Hamman --- docs/installation.rst | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/docs/installation.rst b/docs/installation.rst index 8553d451cb..35865c764d 100644 --- a/docs/installation.rst +++ b/docs/installation.rst @@ -6,6 +6,11 @@ Zarr depends on NumPy. It is generally best to `install NumPy appropriate for your operating system and Python distribution. Other dependencies should be installed automatically if using one of the installation methods below. +Note: Zarr has endorsed `Scientific-Python SPEC 0 `_ and now follows the version support window as outlined below: + +- Python: 36 months after initial release +- Core package dependencies (e.g. NumPy): 24 months after initial release + Install Zarr from PyPI:: $ pip install zarr From f58065b221452acd70235902ad59d920da6fb02f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 11 Mar 2024 08:47:15 -0700 Subject: [PATCH 055/130] Bump pypa/gh-action-pypi-publish from 1.8.12 to 1.8.14 (#1700) Bumps [pypa/gh-action-pypi-publish](https://github.com/pypa/gh-action-pypi-publish) from 1.8.12 to 1.8.14. - [Release notes](https://github.com/pypa/gh-action-pypi-publish/releases) - [Commits](https://github.com/pypa/gh-action-pypi-publish/compare/v1.8.12...v1.8.14) --- updated-dependencies: - dependency-name: pypa/gh-action-pypi-publish dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/releases.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/releases.yml b/.github/workflows/releases.yml index 6d417042b5..fe168d2862 100644 --- a/.github/workflows/releases.yml +++ b/.github/workflows/releases.yml @@ -64,7 +64,7 @@ jobs: with: name: releases path: dist - - uses: pypa/gh-action-pypi-publish@v1.8.12 + - uses: pypa/gh-action-pypi-publish@v1.8.14 with: user: __token__ password: ${{ secrets.pypi_password }} From bbac25472e0a781dc5c0256d26a481eacb27390b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 13 Mar 2024 16:58:21 +0100 Subject: [PATCH 056/130] Bump pytest-doctestplus from 1.2.0 to 1.2.1 (#1699) Bumps [pytest-doctestplus](https://github.com/scientific-python/pytest-doctestplus) from 1.2.0 to 1.2.1. - [Release notes](https://github.com/scientific-python/pytest-doctestplus/releases) - [Changelog](https://github.com/scientific-python/pytest-doctestplus/blob/main/CHANGES.rst) - [Commits](https://github.com/scientific-python/pytest-doctestplus/compare/v1.2.0...v1.2.1) --- updated-dependencies: - dependency-name: pytest-doctestplus dependency-type: direct:development update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Sanket Verma --- requirements_dev_optional.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements_dev_optional.txt b/requirements_dev_optional.txt index b14381dd6e..62b257ea70 100644 --- a/requirements_dev_optional.txt +++ b/requirements_dev_optional.txt @@ -15,7 +15,7 @@ pymongo==4.6.2 # optional test requirements coverage pytest-cov==4.1.0 -pytest-doctestplus==1.2.0 +pytest-doctestplus==1.2.1 pytest-timeout==2.3.1 h5py==3.10.0 fsspec==2023.12.2 From 6fe553df925c224fcc0a12ecdd074997ce9e56f7 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 13 Mar 2024 17:53:29 +0100 Subject: [PATCH 057/130] Bump redis from 5.0.2 to 5.0.3 (#1698) Bumps [redis](https://github.com/redis/redis-py) from 5.0.2 to 5.0.3. - [Release notes](https://github.com/redis/redis-py/releases) - [Changelog](https://github.com/redis/redis-py/blob/master/CHANGES) - [Commits](https://github.com/redis/redis-py/compare/v5.0.2...v5.0.3) --- updated-dependencies: - dependency-name: redis dependency-type: direct:development update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Sanket Verma --- requirements_dev_optional.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements_dev_optional.txt b/requirements_dev_optional.txt index 62b257ea70..7ff673cebd 100644 --- a/requirements_dev_optional.txt +++ b/requirements_dev_optional.txt @@ -8,7 +8,7 @@ ipywidgets==8.1.2 # don't let pyup change pinning for azure-storage-blob, need to pin to older # version to get compatibility with azure storage emulator on appveyor (FIXME) azure-storage-blob==12.16.0 # pyup: ignore -redis==5.0.2 +redis==5.0.3 types-redis types-setuptools pymongo==4.6.2 From f4f0b42d5ced9f777709fb87ff06ff09fbaa3055 Mon Sep 17 00:00:00 2001 From: Joe Hamman Date: Mon, 25 Mar 2024 09:44:40 -0700 Subject: [PATCH 058/130] Add Python 3.12 to CI (#1719) * Update python-package.yml * bump numpy versions * bump min python version * Update release.rst --- .github/workflows/python-package.yml | 10 ++++++---- docs/release.rst | 6 ++++++ pyproject.toml | 2 +- 3 files changed, 13 insertions(+), 5 deletions(-) diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index fd2603ff95..2f9166ae96 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -15,13 +15,15 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: ['3.9', '3.10', '3.11'] - numpy_version: ['>=1.22.0', '==1.21.*'] + python-version: ['3.9', '3.10', '3.11', '3.12'] + numpy_version: ['>=1.24.0', '==1.23.*'] exclude: - python-version: '3.10' - numpy_version: '==1.21.*' + numpy_version: '==1.23.*' - python-version: '3.11' - numpy_version: '==1.21.*' + numpy_version: '==1.23.*' + - python-version: '3.12' + numpy_version: '==1.23.*' services: redis: image: redis diff --git a/docs/release.rst b/docs/release.rst index 037432ca58..5c4da710b2 100644 --- a/docs/release.rst +++ b/docs/release.rst @@ -53,6 +53,12 @@ Maintenance * Fix RTD build. By :user:`Sanket Verma ` :issue:`1694`. +* Add CI test environment for Python 3.12 + By :user:`Joe Hamman ` :issue:`1719`. + +* Bump minimum supported NumPy version to 1.23 (per spec 0000) + By :user:`Joe Hamman ` :issue:`1719`. + .. _release_2.17.0: 2.17.0 diff --git a/pyproject.toml b/pyproject.toml index 4da3079808..0be79f990e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -13,7 +13,7 @@ maintainers = [ requires-python = ">=3.9" dependencies = [ 'asciitree', - 'numpy>=1.21.1', + 'numpy>=1.23', 'fasteners; sys_platform != "emscripten"', 'numcodecs>=0.10.0', ] From 04e862cfcaf58aeb5e13e40b66007f44caa9bcae Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 25 Mar 2024 19:02:07 +0100 Subject: [PATCH 059/130] Bump pytest-cov from 4.1.0 to 5.0.0 (#1722) Bumps [pytest-cov](https://github.com/pytest-dev/pytest-cov) from 4.1.0 to 5.0.0. - [Changelog](https://github.com/pytest-dev/pytest-cov/blob/master/CHANGELOG.rst) - [Commits](https://github.com/pytest-dev/pytest-cov/compare/v4.1.0...v5.0.0) --- updated-dependencies: - dependency-name: pytest-cov dependency-type: direct:development update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- requirements_dev_optional.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements_dev_optional.txt b/requirements_dev_optional.txt index 7ff673cebd..a3411acd67 100644 --- a/requirements_dev_optional.txt +++ b/requirements_dev_optional.txt @@ -14,7 +14,7 @@ types-setuptools pymongo==4.6.2 # optional test requirements coverage -pytest-cov==4.1.0 +pytest-cov==5.0.0 pytest-doctestplus==1.2.1 pytest-timeout==2.3.1 h5py==3.10.0 From 0e58b79dd38911a80648f5a4cbf01439fe90c7a5 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 26 Mar 2024 00:27:27 +0530 Subject: [PATCH 060/130] chore: update pre-commit hooks (#1708) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit updates: - [github.com/astral-sh/ruff-pre-commit: v0.2.2 → v0.3.3](https://github.com/astral-sh/ruff-pre-commit/compare/v0.2.2...v0.3.3) - [github.com/psf/black: 24.2.0 → 24.3.0](https://github.com/psf/black/compare/24.2.0...24.3.0) - [github.com/pre-commit/mirrors-mypy: v1.8.0 → v1.9.0](https://github.com/pre-commit/mirrors-mypy/compare/v1.8.0...v1.9.0) Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Co-authored-by: Sanket Verma --- .pre-commit-config.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 41b65f1d02..46aadb554b 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -8,11 +8,11 @@ default_language_version: repos: - repo: https://github.com/astral-sh/ruff-pre-commit # Ruff version. - rev: 'v0.2.2' + rev: 'v0.3.3' hooks: - id: ruff - repo: https://github.com/psf/black - rev: 24.2.0 + rev: 24.3.0 hooks: - id: black - repo: https://github.com/codespell-project/codespell @@ -24,7 +24,7 @@ repos: hooks: - id: check-yaml - repo: https://github.com/pre-commit/mirrors-mypy - rev: v1.8.0 + rev: v1.9.0 hooks: - id: mypy files: zarr From 8d0d910f27453157558d201e7a6a84bb565c87ad Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 25 Mar 2024 13:46:34 -0700 Subject: [PATCH 061/130] chore: update pre-commit hooks (#1723) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit updates: - [github.com/astral-sh/ruff-pre-commit: v0.3.3 → v0.3.4](https://github.com/astral-sh/ruff-pre-commit/compare/v0.3.3...v0.3.4) Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 46aadb554b..6c2762f34d 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -8,7 +8,7 @@ default_language_version: repos: - repo: https://github.com/astral-sh/ruff-pre-commit # Ruff version. - rev: 'v0.3.3' + rev: 'v0.3.4' hooks: - id: ruff - repo: https://github.com/psf/black From 2534413e2f8b56d9c64744419628a464d639f1dc Mon Sep 17 00:00:00 2001 From: Joe Hamman Date: Tue, 26 Mar 2024 18:44:50 -0700 Subject: [PATCH 062/130] Fix release notes (following #1719) (#1725) --- docs/release.rst | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/docs/release.rst b/docs/release.rst index 5c4da710b2..116393d417 100644 --- a/docs/release.rst +++ b/docs/release.rst @@ -18,6 +18,15 @@ Release notes Unreleased ---------- +Maintenance +~~~~~~~~~~~ + +* Add CI test environment for Python 3.12 + By :user:`Joe Hamman ` :issue:`1719`. + +* Bump minimum supported NumPy version to 1.23 (per spec 0000) + By :user:`Joe Hamman ` :issue:`1719`. + .. _release_2.17.1: 2.17.1 @@ -53,12 +62,6 @@ Maintenance * Fix RTD build. By :user:`Sanket Verma ` :issue:`1694`. -* Add CI test environment for Python 3.12 - By :user:`Joe Hamman ` :issue:`1719`. - -* Bump minimum supported NumPy version to 1.23 (per spec 0000) - By :user:`Joe Hamman ` :issue:`1719`. - .. _release_2.17.0: 2.17.0 From bad8dd0240861c48e768c265911aec2aec24481c Mon Sep 17 00:00:00 2001 From: Deepak Cherian Date: Wed, 27 Mar 2024 12:42:33 -0600 Subject: [PATCH 063/130] Override ipython repr methods. (#1724) Closes #1716 This avoids expensive lookups against object stores. --- docs/release.rst | 6 ++++++ zarr/hierarchy.py | 41 ++++++++++++++++++++++++++++++++++++ zarr/tests/test_hierarchy.py | 21 ++++++++++++++++++ 3 files changed, 68 insertions(+) diff --git a/docs/release.rst b/docs/release.rst index 116393d417..fd48a53b38 100644 --- a/docs/release.rst +++ b/docs/release.rst @@ -18,6 +18,12 @@ Release notes Unreleased ---------- +Enhancements +~~~~~~~~~~~~ + +* Override IPython ``_repr_*_`` methods to avoid expensive lookups against object stores. + By :user:`Deepak Cherian ` :issue:`1716`. + Maintenance ~~~~~~~~~~~ diff --git a/zarr/hierarchy.py b/zarr/hierarchy.py index 44af1d63d1..c88892c932 100644 --- a/zarr/hierarchy.py +++ b/zarr/hierarchy.py @@ -515,6 +515,13 @@ def _delitem_nosync(self, item): raise KeyError(item) def __getattr__(self, item): + # https://github.com/jupyter/notebook/issues/2014 + # Save a possibly expensive lookup (for e.g. against cloud stores) + # Note: The _ipython_display_ method is required to display the right info as a side-effect. + # It is simpler to pretend it doesn't exist. + if item in ["_ipython_canary_method_should_not_exist_", "_ipython_display_"]: + raise AttributeError + # allow access to group members via dot notation try: return self.__getitem__(item) @@ -1331,6 +1338,40 @@ def move(self, source, dest): self._write_op(self._move_nosync, source, dest) + # Override ipython repr methods, GH1716 + # https://ipython.readthedocs.io/en/stable/config/integrating.html#custom-methods + # " If the methods don’t exist, the standard repr() is used. If a method exists and + # returns None, it is treated the same as if it does not exist." + def _repr_html_(self): + return None + + def _repr_latex_(self): + return None + + def _repr_mimebundle_(self, **kwargs): + return None + + def _repr_svg_(self): + return None + + def _repr_png_(self): + return None + + def _repr_jpeg_(self): + return None + + def _repr_markdown_(self): + return None + + def _repr_javascript_(self): + return None + + def _repr_pdf_(self): + return None + + def _repr_json_(self): + return None + def _normalize_store_arg(store, *, storage_options=None, mode="r", zarr_version=None): if zarr_version is None: diff --git a/zarr/tests/test_hierarchy.py b/zarr/tests/test_hierarchy.py index 6c08d7b88a..161e1eb813 100644 --- a/zarr/tests/test_hierarchy.py +++ b/zarr/tests/test_hierarchy.py @@ -1,4 +1,5 @@ import atexit +import operator import os import sys import pickle @@ -87,6 +88,26 @@ def create_group( ) return g + def test_ipython_repr_methods(self): + g = self.create_group() + for method in [ + "html", + "json", + "javascript", + "markdown", + "svg", + "png", + "jpeg", + "latex", + "pdf", + "mimebundle", + ]: + assert operator.methodcaller(f"_repr_{method}_")(g) is None + with pytest.raises(AttributeError): + g._ipython_display_() + with pytest.raises(AttributeError): + g._ipython_canary_method_should_not_exist_() + def test_group_init_1(self): store, chunk_store = self.create_store() g = self.create_group(store, chunk_store=chunk_store) From 37e0a1a0c22f552daf6fd94fec5474a9b92db33d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 28 Mar 2024 16:40:01 +0530 Subject: [PATCH 064/130] Bump pymongo from 4.6.2 to 4.6.3 (#1729) --- requirements_dev_optional.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements_dev_optional.txt b/requirements_dev_optional.txt index a3411acd67..809d1c0eee 100644 --- a/requirements_dev_optional.txt +++ b/requirements_dev_optional.txt @@ -11,7 +11,7 @@ azure-storage-blob==12.16.0 # pyup: ignore redis==5.0.3 types-redis types-setuptools -pymongo==4.6.2 +pymongo==4.6.3 # optional test requirements coverage pytest-cov==5.0.0 From cb5b77a7fcf8312bc28d7970a31395c5690f5898 Mon Sep 17 00:00:00 2001 From: Sanket Verma Date: Fri, 29 Mar 2024 05:37:50 +0530 Subject: [PATCH 065/130] Remove v1 and v2 specification (#1582) * Remove v1 and v2 specification * fix warning --------- Co-authored-by: Davis Bennett --- docs/release.rst | 4 +- docs/spec/v1.rst | 267 +--------------------- docs/spec/v2.rst | 562 +---------------------------------------------- docs/spec/v3.rst | 2 +- 4 files changed, 7 insertions(+), 828 deletions(-) diff --git a/docs/release.rst b/docs/release.rst index fd48a53b38..9c75dc4feb 100644 --- a/docs/release.rst +++ b/docs/release.rst @@ -1625,11 +1625,11 @@ Bug fixes Documentation ~~~~~~~~~~~~~ -* Some changes have been made to the :ref:`spec_v2` document to clarify +* Some changes have been made to the Zarr Specification v2 document to clarify ambiguities and add some missing information. These changes do not break compatibility with any of the material as previously implemented, and so the changes have been made in-place in the document without incrementing the document version number. See the - section on :ref:`spec_v2_changes` in the specification document for more information. + section on changes in the specification document for more information. * A new :ref:`tutorial_indexing` section has been added to the tutorial. * A new :ref:`tutorial_strings` section has been added to the tutorial (:issue:`135`, :issue:`175`). diff --git a/docs/spec/v1.rst b/docs/spec/v1.rst index 13f68ef36e..27a0490e0a 100644 --- a/docs/spec/v1.rst +++ b/docs/spec/v1.rst @@ -3,268 +3,5 @@ Zarr Storage Specification Version 1 ==================================== -This document provides a technical specification of the protocol and -format used for storing a Zarr array. The key words "MUST", "MUST -NOT", "REQUIRED", "SHALL", "SHALL NOT", "SHOULD", "SHOULD NOT", -"RECOMMENDED", "MAY", and "OPTIONAL" in this document are to be -interpreted as described in `RFC 2119 -`_. - -Status ------- - -This specification is deprecated. See :ref:`spec` for the latest version. - -Storage -------- - -A Zarr array can be stored in any storage system that provides a -key/value interface, where a key is an ASCII string and a value is an -arbitrary sequence of bytes, and the supported operations are read -(get the sequence of bytes associated with a given key), write (set -the sequence of bytes associated with a given key) and delete (remove -a key/value pair). - -For example, a directory in a file system can provide this interface, -where keys are file names, values are file contents, and files can be -read, written or deleted via the operating system. Equally, an S3 -bucket can provide this interface, where keys are resource names, -values are resource contents, and resources can be read, written or -deleted via HTTP. - -Below an "array store" refers to any system implementing this -interface. - -Metadata --------- - -Each array requires essential configuration metadata to be stored, -enabling correct interpretation of the stored data. This metadata is -encoded using JSON and stored as the value of the 'meta' key within an -array store. - -The metadata resource is a JSON object. The following keys MUST be -present within the object: - -zarr_format - An integer defining the version of the storage specification to which the - array store adheres. -shape - A list of integers defining the length of each dimension of the array. -chunks - A list of integers defining the length of each dimension of a chunk of the - array. Note that all chunks within a Zarr array have the same shape. -dtype - A string or list defining a valid data type for the array. See also - the subsection below on data type encoding. -compression - A string identifying the primary compression library used to compress - each chunk of the array. -compression_opts - An integer, string or dictionary providing options to the primary - compression library. -fill_value - A scalar value providing the default value to use for uninitialized - portions of the array. -order - Either 'C' or 'F', defining the layout of bytes within each chunk of the - array. 'C' means row-major order, i.e., the last dimension varies fastest; - 'F' means column-major order, i.e., the first dimension varies fastest. - -Other keys MAY be present within the metadata object however they MUST -NOT alter the interpretation of the required fields defined above. - -For example, the JSON object below defines a 2-dimensional array of -64-bit little-endian floating point numbers with 10000 rows and 10000 -columns, divided into chunks of 1000 rows and 1000 columns (so there -will be 100 chunks in total arranged in a 10 by 10 grid). Within each -chunk the data are laid out in C contiguous order, and each chunk is -compressed using the Blosc compression library:: - - { - "chunks": [ - 1000, - 1000 - ], - "compression": "blosc", - "compression_opts": { - "clevel": 5, - "cname": "lz4", - "shuffle": 1 - }, - "dtype": "`_. The -format consists of 3 parts: a character describing the byteorder of -the data (``<``: little-endian, ``>``: big-endian, ``|``: -not-relevant), a character code giving the basic type of the array, -and an integer providing the number of bytes the type uses. The byte -order MUST be specified. E.g., ``"i4"``, ``"|b1"`` and -``"|S12"`` are valid data types. - -Structure data types (i.e., with multiple named fields) are encoded as -a list of two-element lists, following `NumPy array protocol type -descriptions (descr) -`_. -For example, the JSON list ``[["r", "|u1"], ["g", "|u1"], ["b", -"|u1"]]`` defines a data type composed of three single-byte unsigned -integers labelled 'r', 'g' and 'b'. - -Chunks ------- - -Each chunk of the array is compressed by passing the raw bytes for the -chunk through the primary compression library to obtain a new sequence -of bytes comprising the compressed chunk data. No header is added to -the compressed bytes or any other modification made. The internal -structure of the compressed bytes will depend on which primary -compressor was used. For example, the `Blosc compressor -`_ -produces a sequence of bytes that begins with a 16-byte header -followed by compressed data. - -The compressed sequence of bytes for each chunk is stored under a key -formed from the index of the chunk within the grid of chunks -representing the array. To form a string key for a chunk, the indices -are converted to strings and concatenated with the period character -('.') separating each index. For example, given an array with shape -(10000, 10000) and chunk shape (1000, 1000) there will be 100 chunks -laid out in a 10 by 10 grid. The chunk with indices (0, 0) provides -data for rows 0-999 and columns 0-999 and is stored under the key -'0.0'; the chunk with indices (2, 4) provides data for rows 2000-2999 -and columns 4000-4999 and is stored under the key '2.4'; etc. - -There is no need for all chunks to be present within an array -store. If a chunk is not present then it is considered to be in an -uninitialized state. An uninitialized chunk MUST be treated as if it -was uniformly filled with the value of the 'fill_value' field in the -array metadata. If the 'fill_value' field is ``null`` then the -contents of the chunk are undefined. - -Note that all chunks in an array have the same shape. If the length of -any array dimension is not exactly divisible by the length of the -corresponding chunk dimension then some chunks will overhang the edge -of the array. The contents of any chunk region falling outside the -array are undefined. - -Attributes ----------- - -Each array can also be associated with custom attributes, which are -simple key/value items with application-specific meaning. Custom -attributes are encoded as a JSON object and stored under the 'attrs' -key within an array store. Even if the attributes are empty, the -'attrs' key MUST be present within an array store. - -For example, the JSON object below encodes three attributes named -'foo', 'bar' and 'baz':: - - { - "foo": 42, - "bar": "apples", - "baz": [1, 2, 3, 4] - } - -Example -------- - -Below is an example of storing a Zarr array, using a directory on the -local file system as storage. - -Initialize the store:: - - >>> import zarr - >>> store = zarr.DirectoryStore('example.zarr') - >>> zarr.init_store(store, shape=(20, 20), chunks=(10, 10), - ... dtype='i4', fill_value=42, compression='zlib', - ... compression_opts=1, overwrite=True) - -No chunks are initialized yet, so only the 'meta' and 'attrs' keys -have been set:: - - >>> import os - >>> sorted(os.listdir('example.zarr')) - ['attrs', 'meta'] - -Inspect the array metadata:: - - >>> print(open('example.zarr/meta').read()) - { - "chunks": [ - 10, - 10 - ], - "compression": "zlib", - "compression_opts": 1, - "dtype": ">> print(open('example.zarr/attrs').read()) - {} - -Set some data:: - - >>> z = zarr.Array(store) - >>> z[0:10, 0:10] = 1 - >>> sorted(os.listdir('example.zarr')) - ['0.0', 'attrs', 'meta'] - -Set some more data:: - - >>> z[0:10, 10:20] = 2 - >>> z[10:20, :] = 3 - >>> sorted(os.listdir('example.zarr')) - ['0.0', '0.1', '1.0', '1.1', 'attrs', 'meta'] - -Manually decompress a single chunk for illustration:: - - >>> import zlib - >>> b = zlib.decompress(open('example.zarr/0.0', 'rb').read()) - >>> import numpy as np - >>> a = np.frombuffer(b, dtype='>> a - array([1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1], dtype=int32) - -Modify the array attributes:: - - >>> z.attrs['foo'] = 42 - >>> z.attrs['bar'] = 'apples' - >>> z.attrs['baz'] = [1, 2, 3, 4] - >>> print(open('example.zarr/attrs').read()) - { - "bar": "apples", - "baz": [ - 1, - 2, - 3, - 4 - ], - "foo": 42 - } +The V1 Specification has been migrated to its website → +https://zarr-specs.readthedocs.io/. diff --git a/docs/spec/v2.rst b/docs/spec/v2.rst index c1e12e1218..deb6d46ce6 100644 --- a/docs/spec/v2.rst +++ b/docs/spec/v2.rst @@ -3,563 +3,5 @@ Zarr Storage Specification Version 2 ==================================== -This document provides a technical specification of the protocol and format -used for storing Zarr arrays. The key words "MUST", "MUST NOT", "REQUIRED", -"SHALL", "SHALL NOT", "SHOULD", "SHOULD NOT", "RECOMMENDED", "MAY", and -"OPTIONAL" in this document are to be interpreted as described in `RFC 2119 -`_. - -Status ------- - -This specification is the latest version. See :ref:`spec` for previous -versions. - -.. _spec_v2_storage: - -Storage -------- - -A Zarr array can be stored in any storage system that provides a key/value -interface, where a key is an ASCII string and a value is an arbitrary sequence -of bytes, and the supported operations are read (get the sequence of bytes -associated with a given key), write (set the sequence of bytes associated with -a given key) and delete (remove a key/value pair). - -For example, a directory in a file system can provide this interface, where -keys are file names, values are file contents, and files can be read, written -or deleted via the operating system. Equally, an S3 bucket can provide this -interface, where keys are resource names, values are resource contents, and -resources can be read, written or deleted via HTTP. - -Below an "array store" refers to any system implementing this interface. - -.. _spec_v2_array: - -Arrays ------- - -.. _spec_v2_array_metadata: - -Metadata -~~~~~~~~ - -Each array requires essential configuration metadata to be stored, enabling -correct interpretation of the stored data. This metadata is encoded using JSON -and stored as the value of the ".zarray" key within an array store. - -The metadata resource is a JSON object. The following keys MUST be present -within the object: - -zarr_format - An integer defining the version of the storage specification to which the - array store adheres. -shape - A list of integers defining the length of each dimension of the array. -chunks - A list of integers defining the length of each dimension of a chunk of the - array. Note that all chunks within a Zarr array have the same shape. -dtype - A string or list defining a valid data type for the array. See also - the subsection below on data type encoding. -compressor - A JSON object identifying the primary compression codec and providing - configuration parameters, or ``null`` if no compressor is to be used. - The object MUST contain an ``"id"`` key identifying the codec to be used. -fill_value - A scalar value providing the default value to use for uninitialized - portions of the array, or ``null`` if no fill_value is to be used. -order - Either "C" or "F", defining the layout of bytes within each chunk of the - array. "C" means row-major order, i.e., the last dimension varies fastest; - "F" means column-major order, i.e., the first dimension varies fastest. -filters - A list of JSON objects providing codec configurations, or ``null`` if no - filters are to be applied. Each codec configuration object MUST contain a - ``"id"`` key identifying the codec to be used. - -The following keys MAY be present within the object: - -dimension_separator - If present, either the string ``"."`` or ``"/"`` defining the separator placed - between the dimensions of a chunk. If the value is not set, then the - default MUST be assumed to be ``"."``, leading to chunk keys of the form "0.0". - Arrays defined with ``"/"`` as the dimension separator can be considered to have - nested, or hierarchical, keys of the form "0/0" that SHOULD where possible - produce a directory-like structure. - -Other keys SHOULD NOT be present within the metadata object and SHOULD be -ignored by implementations. - -For example, the JSON object below defines a 2-dimensional array of 64-bit -little-endian floating point numbers with 10000 rows and 10000 columns, divided -into chunks of 1000 rows and 1000 columns (so there will be 100 chunks in total -arranged in a 10 by 10 grid). Within each chunk the data are laid out in C -contiguous order. Each chunk is encoded using a delta filter and compressed -using the Blosc compression library prior to storage:: - - { - "chunks": [ - 1000, - 1000 - ], - "compressor": { - "id": "blosc", - "cname": "lz4", - "clevel": 5, - "shuffle": 1 - }, - "dtype": "`. The format -consists of 3 parts: - -* One character describing the byteorder of the data (``"<"``: little-endian; - ``">"``: big-endian; ``"|"``: not-relevant) -* One character code giving the basic type of the array (``"b"``: Boolean (integer - type where all values are only True or False); ``"i"``: integer; ``"u"``: unsigned - integer; ``"f"``: floating point; ``"c"``: complex floating point; ``"m"``: timedelta; - ``"M"``: datetime; ``"S"``: string (fixed-length sequence of char); ``"U"``: unicode - (fixed-length sequence of Py_UNICODE); ``"V"``: other (void * – each item is a - fixed-size chunk of memory)) -* An integer specifying the number of bytes the type uses. - -The byte order MUST be specified. E.g., ``"i4"``, ``"|b1"`` and -``"|S12"`` are valid data type encodings. - -For datetime64 ("M") and timedelta64 ("m") data types, these MUST also include the -units within square brackets. A list of valid units and their definitions are given in -the :ref:`NumPy documentation on Datetimes and Timedeltas -`. -For example, ``"`. Each -sub-list has the form ``[fieldname, datatype, shape]`` where ``shape`` -is optional. ``fieldname`` is a string, ``datatype`` is a string -specifying a simple data type (see above), and ``shape`` is a list of -integers specifying subarray shape. For example, the JSON list below -defines a data type composed of three single-byte unsigned integer -fields named "r", "g" and "b":: - - [["r", "|u1"], ["g", "|u1"], ["b", "|u1"]] - -For example, the JSON list below defines a data type composed of three -fields named "x", "y" and "z", where "x" and "y" each contain 32-bit -floats, and each item in "z" is a 2 by 2 array of floats:: - - [["x", "`_ -produces a sequence of bytes that begins with a 16-byte header followed by -compressed data. - -The compressed sequence of bytes for each chunk is stored under a key formed -from the index of the chunk within the grid of chunks representing the array. -To form a string key for a chunk, the indices are converted to strings and -concatenated with the period character (".") separating each index. For -example, given an array with shape (10000, 10000) and chunk shape (1000, 1000) -there will be 100 chunks laid out in a 10 by 10 grid. The chunk with indices -(0, 0) provides data for rows 0-999 and columns 0-999 and is stored under the -key "0.0"; the chunk with indices (2, 4) provides data for rows 2000-2999 and -columns 4000-4999 and is stored under the key "2.4"; etc. - -There is no need for all chunks to be present within an array store. If a chunk -is not present then it is considered to be in an uninitialized state. An -uninitialized chunk MUST be treated as if it was uniformly filled with the value -of the "fill_value" field in the array metadata. If the "fill_value" field is -``null`` then the contents of the chunk are undefined. - -Note that all chunks in an array have the same shape. If the length of any -array dimension is not exactly divisible by the length of the corresponding -chunk dimension then some chunks will overhang the edge of the array. The -contents of any chunk region falling outside the array are undefined. - -.. _spec_v2_array_filters: - -Filters -~~~~~~~ - -Optionally a sequence of one or more filters can be used to transform chunk -data prior to compression. When storing data, filters are applied in the order -specified in array metadata to encode data, then the encoded data are passed to -the primary compressor. When retrieving data, stored chunk data are -decompressed by the primary compressor then decoded using filters in the -reverse order. - -.. _spec_v2_hierarchy: - -Hierarchies ------------ - -.. _spec_v2_hierarchy_paths: - -Logical storage paths -~~~~~~~~~~~~~~~~~~~~~ - -Multiple arrays can be stored in the same array store by associating each array -with a different logical path. A logical path is simply an ASCII string. The -logical path is used to form a prefix for keys used by the array. For example, -if an array is stored at logical path "foo/bar" then the array metadata will be -stored under the key "foo/bar/.zarray", the user-defined attributes will be -stored under the key "foo/bar/.zattrs", and the chunks will be stored under -keys like "foo/bar/0.0", "foo/bar/0.1", etc. - -To ensure consistent behaviour across different storage systems, logical paths -MUST be normalized as follows: - -* Replace all backward slash characters ("\\\\") with forward slash characters - ("/") -* Strip any leading "/" characters -* Strip any trailing "/" characters -* Collapse any sequence of more than one "/" character into a single "/" - character - -The key prefix is then obtained by appending a single "/" character to the -normalized logical path. - -After normalization, if splitting a logical path by the "/" character results -in any path segment equal to the string "." or the string ".." then an error -MUST be raised. - -N.B., how the underlying array store processes requests to store values under -keys containing the "/" character is entirely up to the store implementation -and is not constrained by this specification. E.g., an array store could simply -treat all keys as opaque ASCII strings; equally, an array store could map -logical paths onto some kind of hierarchical storage (e.g., directories on a -file system). - -.. _spec_v2_hierarchy_groups: - -Groups -~~~~~~ - -Arrays can be organized into groups which can also contain other groups. A -group is created by storing group metadata under the ".zgroup" key under some -logical path. E.g., a group exists at the root of an array store if the -".zgroup" key exists in the store, and a group exists at logical path "foo/bar" -if the "foo/bar/.zgroup" key exists in the store. - -If the user requests a group to be created under some logical path, then groups -MUST also be created at all ancestor paths. E.g., if the user requests group -creation at path "foo/bar" then groups MUST be created at path "foo" and the -root of the store, if they don't already exist. - -If the user requests an array to be created under some logical path, then -groups MUST also be created at all ancestor paths. E.g., if the user requests -array creation at path "foo/bar/baz" then groups must be created at path -"foo/bar", path "foo", and the root of the store, if they don't already exist. - -The group metadata resource is a JSON object. The following keys MUST be present -within the object: - -zarr_format - An integer defining the version of the storage specification to which the - array store adheres. - -Other keys MUST NOT be present within the metadata object. - -The members of a group are arrays and groups stored under logical paths that -are direct children of the parent group's logical path. E.g., if groups exist -under the logical paths "foo" and "foo/bar" and an array exists at logical path -"foo/baz" then the members of the group at path "foo" are the group at path -"foo/bar" and the array at path "foo/baz". - -.. _spec_v2_attrs: - -Attributes ----------- - -An array or group can be associated with custom attributes, which are arbitrary -key/value pairs with application-specific meaning. Custom attributes are encoded -as a JSON object and stored under the ".zattrs" key within an array store. The -".zattrs" key does not have to be present, and if it is absent the attributes -should be treated as empty. - -For example, the JSON object below encodes three attributes named -"foo", "bar" and "baz":: - - { - "foo": 42, - "bar": "apples", - "baz": [1, 2, 3, 4] - } - -.. _spec_v2_examples: - -Examples --------- - -Storing a single array -~~~~~~~~~~~~~~~~~~~~~~ - -Below is an example of storing a Zarr array, using a directory on the -local file system as storage. - -Create an array:: - - >>> import zarr - >>> store = zarr.DirectoryStore('data/example.zarr') - >>> a = zarr.create(shape=(20, 20), chunks=(10, 10), dtype='i4', - ... fill_value=42, compressor=zarr.Zlib(level=1), - ... store=store, overwrite=True) - -No chunks are initialized yet, so only the ".zarray" and ".zattrs" keys -have been set in the store:: - - >>> import os - >>> sorted(os.listdir('data/example.zarr')) - ['.zarray'] - -Inspect the array metadata:: - - >>> print(open('data/example.zarr/.zarray').read()) - { - "chunks": [ - 10, - 10 - ], - "compressor": { - "id": "zlib", - "level": 1 - }, - "dtype": ">> a[0:10, 0:10] = 1 - >>> sorted(os.listdir('data/example.zarr')) - ['.zarray', '0.0'] - -Set some more data:: - - >>> a[0:10, 10:20] = 2 - >>> a[10:20, :] = 3 - >>> sorted(os.listdir('data/example.zarr')) - ['.zarray', '0.0', '0.1', '1.0', '1.1'] - -Manually decompress a single chunk for illustration:: - - >>> import zlib - >>> buf = zlib.decompress(open('data/example.zarr/0.0', 'rb').read()) - >>> import numpy as np - >>> chunk = np.frombuffer(buf, dtype='>> chunk - array([1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1], dtype=int32) - -Modify the array attributes:: - - >>> a.attrs['foo'] = 42 - >>> a.attrs['bar'] = 'apples' - >>> a.attrs['baz'] = [1, 2, 3, 4] - >>> sorted(os.listdir('data/example.zarr')) - ['.zarray', '.zattrs', '0.0', '0.1', '1.0', '1.1'] - >>> print(open('data/example.zarr/.zattrs').read()) - { - "bar": "apples", - "baz": [ - 1, - 2, - 3, - 4 - ], - "foo": 42 - } - -Storing multiple arrays in a hierarchy -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -Below is an example of storing multiple Zarr arrays organized into a group -hierarchy, using a directory on the local file system as storage. This storage -implementation maps logical paths onto directory paths on the file system, -however this is an implementation choice and is not required. - -Setup the store:: - - >>> import zarr - >>> store = zarr.DirectoryStore('data/group.zarr') - -Create the root group:: - - >>> root_grp = zarr.group(store, overwrite=True) - -The metadata resource for the root group has been created:: - - >>> import os - >>> sorted(os.listdir('data/group.zarr')) - ['.zgroup'] - -Inspect the group metadata:: - - >>> print(open('data/group.zarr/.zgroup').read()) - { - "zarr_format": 2 - } - -Create a sub-group:: - - >>> sub_grp = root_grp.create_group('foo') - -What has been stored:: - - >>> sorted(os.listdir('data/group.zarr')) - ['.zgroup', 'foo'] - >>> sorted(os.listdir('data/group.zarr/foo')) - ['.zgroup'] - -Create an array within the sub-group:: - - >>> a = sub_grp.create_dataset('bar', shape=(20, 20), chunks=(10, 10)) - >>> a[:] = 42 - -Set a custom attributes:: - - >>> a.attrs['comment'] = 'answer to life, the universe and everything' - -What has been stored:: - - >>> sorted(os.listdir('data/group.zarr')) - ['.zgroup', 'foo'] - >>> sorted(os.listdir('data/group.zarr/foo')) - ['.zgroup', 'bar'] - >>> sorted(os.listdir('data/group.zarr/foo/bar')) - ['.zarray', '.zattrs', '0.0', '0.1', '1.0', '1.1'] - -Here is the same example using a Zip file as storage:: - - >>> store = zarr.ZipStore('data/group.zip', mode='w') - >>> root_grp = zarr.group(store) - >>> sub_grp = root_grp.create_group('foo') - >>> a = sub_grp.create_dataset('bar', shape=(20, 20), chunks=(10, 10)) - >>> a[:] = 42 - >>> a.attrs['comment'] = 'answer to life, the universe and everything' - >>> store.close() - -What has been stored:: - - >>> import zipfile - >>> zf = zipfile.ZipFile('data/group.zip', mode='r') - >>> for name in sorted(zf.namelist()): - ... print(name) - .zgroup - foo/.zgroup - foo/bar/.zarray - foo/bar/.zattrs - foo/bar/0.0 - foo/bar/0.1 - foo/bar/1.0 - foo/bar/1.1 - -.. _spec_v2_changes: - -Changes -------- - -Version 2 clarifications -~~~~~~~~~~~~~~~~~~~~~~~~ - -The following changes have been made to the version 2 specification since it was -initially published to clarify ambiguities and add some missing information. - -* The specification now describes how bytes fill values should be encoded and - decoded for arrays with a fixed-length byte string data type (:issue:`165`, - :issue:`176`). - -* The specification now clarifies that units must be specified for datetime64 and - timedelta64 data types (:issue:`85`, :issue:`215`). - -* The specification now clarifies that the '.zattrs' key does not have to be present for - either arrays or groups, and if absent then custom attributes should be treated as - empty. - -* The specification now describes how structured datatypes with - subarray shapes and/or with nested structured data types are encoded - in array metadata (:issue:`111`, :issue:`296`). - -* Clarified the key/value pairs of custom attributes as "arbitrary" rather than - "simple". - -Changes from version 1 to version 2 -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -The following changes were made between version 1 and version 2 of this specification: - -* Added support for storing multiple arrays in the same store and organising - arrays into hierarchies using groups. -* Array metadata is now stored under the ".zarray" key instead of the "meta" - key. -* Custom attributes are now stored under the ".zattrs" key instead of the - "attrs" key. -* Added support for filters. -* Changed encoding of "fill_value" field within array metadata. -* Changed encoding of compressor information within array metadata to be - consistent with representation of filter information. +The V2 Specification has been migrated to its website → +https://zarr-specs.readthedocs.io/. diff --git a/docs/spec/v3.rst b/docs/spec/v3.rst index bd8852707b..3d39f35ba6 100644 --- a/docs/spec/v3.rst +++ b/docs/spec/v3.rst @@ -1,7 +1,7 @@ .. _spec_v3: Zarr Storage Specification Version 3 -======================================================= +==================================== The V3 Specification has been migrated to its website → https://zarr-specs.readthedocs.io/. From 97cb3a7361ab763d6c2a7ab1128780f9142bd6aa Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 1 Apr 2024 23:46:31 +0200 Subject: [PATCH 066/130] chore: update pre-commit hooks (#1738) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit updates: - [github.com/astral-sh/ruff-pre-commit: v0.3.4 → v0.3.5](https://github.com/astral-sh/ruff-pre-commit/compare/v0.3.4...v0.3.5) Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 6c2762f34d..24ff72a12f 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -8,7 +8,7 @@ default_language_version: repos: - repo: https://github.com/astral-sh/ruff-pre-commit # Ruff version. - rev: 'v0.3.4' + rev: 'v0.3.5' hooks: - id: ruff - repo: https://github.com/psf/black From 6ef6714bc63d66c409cf5780791d6349081fcd45 Mon Sep 17 00:00:00 2001 From: Deepak Cherian Date: Mon, 1 Apr 2024 23:13:42 -0600 Subject: [PATCH 067/130] Optimize Array.info and Group.info (#1733) * Optimize Array.info. Avoid repeated computes of the same value (getsize) * Don't have InfoReporter query items twice. Apparently IPython will run both __repr__ and _repr_html_ so we were calling `getsize` twice. * Group too * Apply suggestions from code review Co-authored-by: Joe Hamman --------- Co-authored-by: Joe Hamman --- docs/release.rst | 2 ++ zarr/core.py | 13 +++++++------ zarr/hierarchy.py | 3 +-- zarr/util.py | 7 +++---- 4 files changed, 13 insertions(+), 12 deletions(-) diff --git a/docs/release.rst b/docs/release.rst index 9c75dc4feb..736838cdaf 100644 --- a/docs/release.rst +++ b/docs/release.rst @@ -21,6 +21,8 @@ Unreleased Enhancements ~~~~~~~~~~~~ +* Optimize ``Array.info`` so that it calls `getsize` only once. + By :user:`Deepak Cherian `. * Override IPython ``_repr_*_`` methods to avoid expensive lookups against object stores. By :user:`Deepak Cherian ` :issue:`1716`. diff --git a/zarr/core.py b/zarr/core.py index c3184c6652..07f38cd96d 100644 --- a/zarr/core.py +++ b/zarr/core.py @@ -176,7 +176,6 @@ def __init__( ) # initialize info reporter - self._info_reporter = InfoReporter(self) # initialize indexing helpers self._oindex = OIndex(self) @@ -2429,7 +2428,7 @@ def info(self): Chunks initialized : 0/10 """ - return self._info_reporter + return InfoReporter(self) def info_items(self): return self._synchronized_op(self._info_items_nosync) @@ -2471,14 +2470,16 @@ def bytestr(n): items += [("Synchronizer type", typestr(self._synchronizer))] # storage info + nbytes = self.nbytes + nbytes_stored = self.nbytes_stored items += [("Store type", typestr(self._store))] if self._chunk_store is not None: items += [("Chunk store type", typestr(self._chunk_store))] - items += [("No. bytes", bytestr(self.nbytes))] - if self.nbytes_stored > 0: + items += [("No. bytes", bytestr(nbytes))] + if nbytes_stored > 0: items += [ - ("No. bytes stored", bytestr(self.nbytes_stored)), - ("Storage ratio", f"{self.nbytes / self.nbytes_stored:.1f}"), + ("No. bytes stored", bytestr(nbytes_stored)), + ("Storage ratio", f"{nbytes / nbytes_stored:.1f}"), ] items += [("Chunks initialized", f"{self.nchunks_initialized}/{self.nchunks}")] diff --git a/zarr/hierarchy.py b/zarr/hierarchy.py index c88892c932..c5f7a37bc6 100644 --- a/zarr/hierarchy.py +++ b/zarr/hierarchy.py @@ -211,7 +211,6 @@ def __init__( ) # setup info - self._info = InfoReporter(self) @property def store(self): @@ -266,7 +265,7 @@ def attrs(self): @property def info(self): """Return diagnostic information about the group.""" - return self._info + return InfoReporter(self) @property def meta_array(self): diff --git a/zarr/util.py b/zarr/util.py index dc8aff0edf..848f1ed114 100644 --- a/zarr/util.py +++ b/zarr/util.py @@ -408,14 +408,13 @@ def info_html_report(items) -> str: class InfoReporter: def __init__(self, obj): self.obj = obj + self.items = self.obj.info_items() def __repr__(self): - items = self.obj.info_items() - return info_text_report(items) + return info_text_report(self.items) def _repr_html_(self): - items = self.obj.info_items() - return info_html_report(items) + return info_html_report(self.items) class TreeNode: From 9864b40731a11ed826f6b69e6b9e381a12168096 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 2 Apr 2024 05:29:11 +0000 Subject: [PATCH 068/130] Bump actions/setup-python from 5.0.0 to 5.1.0 (#1736) Bumps [actions/setup-python](https://github.com/actions/setup-python) from 5.0.0 to 5.1.0. - [Release notes](https://github.com/actions/setup-python/releases) - [Commits](https://github.com/actions/setup-python/compare/v5.0.0...v5.1.0) --- updated-dependencies: - dependency-name: actions/setup-python dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/releases.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/releases.yml b/.github/workflows/releases.yml index fe168d2862..8ac76c899b 100644 --- a/.github/workflows/releases.yml +++ b/.github/workflows/releases.yml @@ -16,7 +16,7 @@ jobs: submodules: true fetch-depth: 0 - - uses: actions/setup-python@v5.0.0 + - uses: actions/setup-python@v5.1.0 name: Install Python with: python-version: '3.9' From aa9a0d5e1874217bcaa15340860cb8b606944fb6 Mon Sep 17 00:00:00 2001 From: Dimitri Papadopoulos Orfanos <3234522+DimitriPapadopoulos@users.noreply.github.com> Date: Tue, 2 Apr 2024 19:45:48 +0200 Subject: [PATCH 069/130] Couple fixes (#1737) * Use `is` when comparing `type` of two objects * Unnecessary `None` provided as default --- zarr/_storage/store.py | 2 +- zarr/tests/test_core.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/zarr/_storage/store.py b/zarr/_storage/store.py index 209f118534..0a08080548 100644 --- a/zarr/_storage/store.py +++ b/zarr/_storage/store.py @@ -462,7 +462,7 @@ def inner_store(self) -> Union["StorageTransformer", StoreV3]: def __eq__(self, other): return ( - type(self) == type(other) + type(self) is type(other) and self._inner_store == other._inner_store and self.get_config() == other.get_config() ) diff --git a/zarr/tests/test_core.py b/zarr/tests/test_core.py index d9447c0832..730f724314 100644 --- a/zarr/tests/test_core.py +++ b/zarr/tests/test_core.py @@ -123,7 +123,7 @@ def create_array(self, shape: Union[int, Tuple[int, ...]], **kwargs): "compressor": kwargs.pop("compressor", self.compressor), "chunk_store": chunk_store, "storage_transformers": self.create_storage_transformers(shape), - "filters": kwargs.pop("filters", self.create_filters(kwargs.get("dtype", None))), + "filters": kwargs.pop("filters", self.create_filters(kwargs.get("dtype"))), } # keyword arguments for array instantiation From d0fb8758483dd811234300eea7a1f24909bb929f Mon Sep 17 00:00:00 2001 From: David Stansby Date: Wed, 3 Apr 2024 20:18:18 +0200 Subject: [PATCH 070/130] Fix tests with Pytest 8 (#1714) * Bump pytest version * Use editable install when testing --------- Co-authored-by: Sanket Verma --- .github/workflows/python-package.yml | 2 +- requirements_dev_minimal.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index 2f9166ae96..a37fa3c63a 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -64,7 +64,7 @@ jobs: python -m pip install --upgrade pip python -m pip install -U pip setuptools wheel line_profiler python -m pip install -rrequirements_dev_minimal.txt numpy${{matrix.numpy_version}} -rrequirements_dev_optional.txt pymongo redis - python -m pip install . + python -m pip install -e . python -m pip freeze - name: Tests shell: "bash -l {0}" diff --git a/requirements_dev_minimal.txt b/requirements_dev_minimal.txt index 94d3fff8a6..5d156db655 100644 --- a/requirements_dev_minimal.txt +++ b/requirements_dev_minimal.txt @@ -5,4 +5,4 @@ numcodecs==0.12.1 msgpack-python==0.5.6 setuptools-scm==8.0.4 # test requirements -pytest==7.4.4 +pytest==8.1.1 From bcb7684b5a15d4ead21e305301581e02193e8677 Mon Sep 17 00:00:00 2001 From: Deepak Cherian Date: Thu, 4 Apr 2024 16:17:08 -0600 Subject: [PATCH 071/130] Avoid redundant __contains__ (#1739) Let's try grabbing the array.json and group.json files, and check for `*NotFoundError`, instead of using contains first. Co-authored-by: Davis Bennett --- zarr/hierarchy.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/zarr/hierarchy.py b/zarr/hierarchy.py index c5f7a37bc6..0067eaebb5 100644 --- a/zarr/hierarchy.py +++ b/zarr/hierarchy.py @@ -27,6 +27,7 @@ from zarr.errors import ( ContainsArrayError, ContainsGroupError, + ArrayNotFoundError, GroupNotFoundError, ReadOnlyError, ) @@ -457,7 +458,7 @@ def __getitem__(self, item): """ path = self._item_path(item) - if contains_array(self._store, path): + try: return Array( self._store, read_only=self._read_only, @@ -468,7 +469,10 @@ def __getitem__(self, item): zarr_version=self._version, meta_array=self._meta_array, ) - elif contains_group(self._store, path, explicit_only=True): + except ArrayNotFoundError: + pass + + try: return Group( self._store, read_only=self._read_only, @@ -479,7 +483,10 @@ def __getitem__(self, item): zarr_version=self._version, meta_array=self._meta_array, ) - elif self._version == 3: + except GroupNotFoundError: + pass + + if self._version == 3: implicit_group = meta_root + path + "/" # non-empty folder in the metadata path implies an implicit group if self._store.list_prefix(implicit_group): From 0cfd2be24feb7cd86002b9ab22cd65f65ed85e3a Mon Sep 17 00:00:00 2001 From: Deepak Cherian Date: Thu, 4 Apr 2024 20:13:23 -0600 Subject: [PATCH 072/130] Array & Group: Use already loaded attributes to populate cache. (#1734) * Array: Use already loaded attributes to populate cache. * Group: Use already loaded attributes to populate cache. * Fix * Add release note --- docs/release.rst | 4 ++++ zarr/attrs.py | 6 ++++-- zarr/core.py | 7 ++++++- zarr/hierarchy.py | 7 ++++++- 4 files changed, 20 insertions(+), 4 deletions(-) diff --git a/docs/release.rst b/docs/release.rst index 736838cdaf..346d673d68 100644 --- a/docs/release.rst +++ b/docs/release.rst @@ -21,8 +21,12 @@ Unreleased Enhancements ~~~~~~~~~~~~ +* [v3] Reuse the download array metadata when creating an ``Array``. + By :user:`Deepak Cherian `. + * Optimize ``Array.info`` so that it calls `getsize` only once. By :user:`Deepak Cherian `. + * Override IPython ``_repr_*_`` methods to avoid expensive lookups against object stores. By :user:`Deepak Cherian ` :issue:`1716`. diff --git a/zarr/attrs.py b/zarr/attrs.py index e967c5b853..2afcaf295e 100644 --- a/zarr/attrs.py +++ b/zarr/attrs.py @@ -25,14 +25,16 @@ class Attributes(MutableMapping): """ - def __init__(self, store, key=".zattrs", read_only=False, cache=True, synchronizer=None): + def __init__( + self, store, key=".zattrs", read_only=False, cache=True, synchronizer=None, cached_dict=None + ): self._version = getattr(store, "_store_version", 2) _Store = Store if self._version == 2 else StoreV3 self.store = _Store._ensure_store(store) self.key = key self.read_only = read_only self.cache = cache - self._cached_asdict = None + self._cached_asdict = cached_dict if cache else None self.synchronizer = synchronizer def _get_nosync(self): diff --git a/zarr/core.py b/zarr/core.py index 07f38cd96d..1bd081acee 100644 --- a/zarr/core.py +++ b/zarr/core.py @@ -172,7 +172,12 @@ def __init__( # initialize attributes akey = _prefix_to_attrs_key(self._store, self._key_prefix) self._attrs = Attributes( - store, key=akey, read_only=read_only, synchronizer=synchronizer, cache=cache_attrs + store, + key=akey, + read_only=read_only, + synchronizer=synchronizer, + cache=cache_attrs, + cached_dict=self._meta["attributes"] if self._version == 3 else None, ) # initialize info reporter diff --git a/zarr/hierarchy.py b/zarr/hierarchy.py index 0067eaebb5..0fb07dd620 100644 --- a/zarr/hierarchy.py +++ b/zarr/hierarchy.py @@ -208,7 +208,12 @@ def __init__( # object can still be created. akey = mkey self._attrs = Attributes( - store, key=akey, read_only=read_only, cache=cache_attrs, synchronizer=synchronizer + store, + key=akey, + read_only=read_only, + cache=cache_attrs, + synchronizer=synchronizer, + cached_dict=self._meta["attributes"] if self._version == 3 and self._meta else None, ) # setup info From 62910bcebd3a583cea79af2077fa1d1798845797 Mon Sep 17 00:00:00 2001 From: Deepak Cherian Date: Thu, 4 Apr 2024 20:14:09 -0600 Subject: [PATCH 073/130] Optimize attribute setting (#1741) * Optimize attribute setting * Add release note --------- Co-authored-by: Davis Bennett --- docs/release.rst | 3 +++ zarr/attrs.py | 7 ++++--- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/docs/release.rst b/docs/release.rst index 346d673d68..b011b0986b 100644 --- a/docs/release.rst +++ b/docs/release.rst @@ -20,6 +20,9 @@ Unreleased Enhancements ~~~~~~~~~~~~ +* [v3] Dramatically reduce number of ``__contains_`` requests in favor of optimistically calling `__getitem__` + and handling any error that may arise. + By :user:`Deepak Cherian `. * [v3] Reuse the download array metadata when creating an ``Array``. By :user:`Deepak Cherian `. diff --git a/zarr/attrs.py b/zarr/attrs.py index 2afcaf295e..af9a5f1d30 100644 --- a/zarr/attrs.py +++ b/zarr/attrs.py @@ -151,19 +151,20 @@ def _put_nosync(self, d): if self.cache: self._cached_asdict = d else: - if self.key in self.store: + try: + meta_unparsed = self.store[self.key] # Cannot write the attributes directly to JSON, but have to # store it within the pre-existing attributes key of the v3 # metadata. # Note: this changes the store.counter result in test_caching_on! - meta = self.store._metadata_class.parse_metadata(self.store[self.key]) + meta = self.store._metadata_class.parse_metadata(meta_unparsed) if "attributes" in meta and "filters" in meta["attributes"]: # need to preserve any existing "filters" attribute d["attributes"]["filters"] = meta["attributes"]["filters"] meta["attributes"] = d["attributes"] - else: + except KeyError: meta = d self.store[self.key] = json_dumps(meta) if self.cache: From 5fde3a29ca0a9f1a005eb5000d1e8585d4ca0bde Mon Sep 17 00:00:00 2001 From: Ian Carroll Date: Fri, 5 Apr 2024 13:15:10 -0400 Subject: [PATCH 074/130] Make sure fs exceptions are raised if not MissingFs exceptions (clone) (#1604) * Make sure fs exceptions are raised if not Missing * lint * add missing argument in tests, lint * clear memory filesystem during test * improve commenting * add memory_store fixture, getitems performance * Update release.rst * improve FSStore.test_exception coverage --------- Co-authored-by: Martin Durant Co-authored-by: Joe Hamman Co-authored-by: Josh Moore Co-authored-by: Sanket Verma --- docs/release.rst | 3 +++ zarr/storage.py | 22 +++++++++++++++++----- zarr/tests/test_storage.py | 25 +++++++++++++++++++++++++ 3 files changed, 45 insertions(+), 5 deletions(-) diff --git a/docs/release.rst b/docs/release.rst index b011b0986b..da802651c2 100644 --- a/docs/release.rst +++ b/docs/release.rst @@ -41,6 +41,9 @@ Maintenance * Bump minimum supported NumPy version to 1.23 (per spec 0000) By :user:`Joe Hamman ` :issue:`1719`. + +* FSStore now raises rather than return bad data. + By :user:`Martin Durant ` and :user:`Ian Carroll ` :issue:`1604`. .. _release_2.17.1: diff --git a/zarr/storage.py b/zarr/storage.py index f6903d29b2..10f55f0ba3 100644 --- a/zarr/storage.py +++ b/zarr/storage.py @@ -1417,11 +1417,23 @@ def _normalize_key(self, key): def getitems( self, keys: Sequence[str], *, contexts: Mapping[str, Context] ) -> Mapping[str, Any]: - keys_transformed = [self._normalize_key(key) for key in keys] - results = self.map.getitems(keys_transformed, on_error="omit") - # The function calling this method may not recognize the transformed keys - # So we send the values returned by self.map.getitems back into the original key space. - return {keys[keys_transformed.index(rk)]: rv for rk, rv in results.items()} + keys_transformed = {self._normalize_key(key): key for key in keys} + results_transformed = self.map.getitems(list(keys_transformed), on_error="return") + results = {} + for k, v in results_transformed.items(): + if isinstance(v, self.exceptions): + # Cause recognized exceptions to prompt a KeyError in the + # function calling this method + continue + elif isinstance(v, Exception): + # Raise any other exception + raise v + else: + # The function calling this method may not recognize the transformed + # keys, so we send the values returned by self.map.getitems back into + # the original key space. + results[keys_transformed[k]] = v + return results def __getitem__(self, key): key = self._normalize_key(key) diff --git a/zarr/tests/test_storage.py b/zarr/tests/test_storage.py index 358d043ad6..ae8a56fa61 100644 --- a/zarr/tests/test_storage.py +++ b/zarr/tests/test_storage.py @@ -1098,6 +1098,12 @@ def mock_walker_no_slash(_path): @pytest.mark.skipif(have_fsspec is False, reason="needs fsspec") class TestFSStore(StoreTests): + @pytest.fixture + def memory_store(self): + store = FSStore("memory://") + yield store + store.fs.store.clear() + def create_store(self, normalize_keys=False, dimension_separator=".", path=None, **kwargs): if path is None: path = tempfile.mkdtemp() @@ -1337,6 +1343,25 @@ def test_s3_complex(self): ) assert (a[:] == -np.ones((8, 8, 8))).all() + def test_exceptions(self, memory_store): + fs = memory_store.fs + group = zarr.open(memory_store, mode="w") + x = group.create_dataset("x", data=[1, 2, 3]) + y = group.create_dataset("y", data=1) + fs.store["/x/0"] = None + fs.store["/y/0"] = None + # no exception from FSStore.getitems getting KeyError + assert group.store.getitems(["foo"], contexts={}) == {} + # exception from FSStore.getitems getting AttributeError + with pytest.raises(Exception): + group.store.getitems(["x/0"], contexts={}) + # exception from FSStore.getitems getting AttributeError + with pytest.raises(Exception): + x[...] + # exception from FSStore.__getitem__ getting AttributeError + with pytest.raises(Exception): + y[...] + @pytest.mark.skipif(have_fsspec is False, reason="needs fsspec") class TestFSStoreWithKeySeparator(StoreTests): From 1631c109c655c71221eab6d964063abd015011f2 Mon Sep 17 00:00:00 2001 From: Joe Hamman Date: Fri, 5 Apr 2024 13:53:33 -0700 Subject: [PATCH 075/130] chore(release): update changelog for 2.17.2 (#1775) --- docs/release.rst | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/docs/release.rst b/docs/release.rst index da802651c2..60d01b8244 100644 --- a/docs/release.rst +++ b/docs/release.rst @@ -13,18 +13,18 @@ Release notes # to document your changes. On releases it will be # re-indented so that it does not show up in the notes. -.. _unreleased: +.. _release_2.17.2: -Unreleased ----------- +2.17.2 +------ Enhancements ~~~~~~~~~~~~ -* [v3] Dramatically reduce number of ``__contains_`` requests in favor of optimistically calling `__getitem__` +* [v3] Dramatically reduce number of ``__contains__`` requests in favor of optimistically calling `__getitem__` and handling any error that may arise. By :user:`Deepak Cherian `. -* [v3] Reuse the download array metadata when creating an ``Array``. +* [v3] Reuse the downloaded array metadata when creating an ``Array``. By :user:`Deepak Cherian `. * Optimize ``Array.info`` so that it calls `getsize` only once. @@ -33,6 +33,9 @@ Enhancements * Override IPython ``_repr_*_`` methods to avoid expensive lookups against object stores. By :user:`Deepak Cherian ` :issue:`1716`. +* FSStore now raises rather than return bad data. + By :user:`Martin Durant ` and :user:`Ian Carroll ` :issue:`1604`. + Maintenance ~~~~~~~~~~~ @@ -42,8 +45,6 @@ Maintenance * Bump minimum supported NumPy version to 1.23 (per spec 0000) By :user:`Joe Hamman ` :issue:`1719`. -* FSStore now raises rather than return bad data. - By :user:`Martin Durant ` and :user:`Ian Carroll ` :issue:`1604`. .. _release_2.17.1: From 6105ef203e3e5c5390aa01db613bc42e8fb98c1a Mon Sep 17 00:00:00 2001 From: Joe Hamman Date: Sat, 6 Apr 2024 11:50:52 -0700 Subject: [PATCH 076/130] chore(docs): reset release notes as unreleased (#1776) --- docs/release.rst | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/docs/release.rst b/docs/release.rst index 60d01b8244..75193bc3e3 100644 --- a/docs/release.rst +++ b/docs/release.rst @@ -13,6 +13,23 @@ Release notes # to document your changes. On releases it will be # re-indented so that it does not show up in the notes. +.. _unreleased: + +Unreleased +---------- + +Enhancements +~~~~~~~~~~~~ + + +Docs +~~~~ + + +Maintenance +~~~~~~~~~~~ + + .. _release_2.17.2: 2.17.2 From 5d566eb16db0f63989816b1300f1ec50927f6632 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 22 Apr 2024 18:58:44 +0200 Subject: [PATCH 077/130] Bump codecov/codecov-action from 3 to 4 (#1647) * Bump codecov/codecov-action from 3 to 4 Bumps [codecov/codecov-action](https://github.com/codecov/codecov-action) from 3 to 4. - [Release notes](https://github.com/codecov/codecov-action/releases) - [Changelog](https://github.com/codecov/codecov-action/blob/main/CHANGELOG.md) - [Commits](https://github.com/codecov/codecov-action/compare/v3...v4) --- updated-dependencies: - dependency-name: codecov/codecov-action dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] * Set codecov env --------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Josh Moore --- .github/workflows/python-package.yml | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index a37fa3c63a..8ff6e9a2eb 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -80,11 +80,8 @@ jobs: mkdir ~/blob_emulator azurite -l ~/blob_emulator --debug debug.log 2>&1 > stdouterr.log & pytest --cov=zarr --cov-config=pyproject.toml --doctest-plus --cov-report xml --cov=./ --timeout=300 - - uses: codecov/codecov-action@v3 + - uses: codecov/codecov-action@v4 + env: + CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} with: - token: ${{ secrets.CODECOV_TOKEN }} # not required for public repos - #files: ./coverage1.xml,./coverage2.xml # optional - #flags: unittests # optional - #name: codecov-umbrella # optional - #fail_ci_if_error: true # optional (default = false) verbose: true # optional (default = false) From b98f6941912da99dcd8350c49a5306e74af05e14 Mon Sep 17 00:00:00 2001 From: Sanket Verma Date: Mon, 22 Apr 2024 21:26:31 +0200 Subject: [PATCH 078/130] Update release.rst for v2.17.2 (#1778) * Update release.rst for v2.17.2 * Minor edits --------- Co-authored-by: Joe Hamman --- docs/release.rst | 29 +++++++++++++++++++++++++---- 1 file changed, 25 insertions(+), 4 deletions(-) diff --git a/docs/release.rst b/docs/release.rst index 75193bc3e3..3b5ab631df 100644 --- a/docs/release.rst +++ b/docs/release.rst @@ -37,15 +37,16 @@ Maintenance Enhancements ~~~~~~~~~~~~ + * [v3] Dramatically reduce number of ``__contains__`` requests in favor of optimistically calling `__getitem__` and handling any error that may arise. - By :user:`Deepak Cherian `. + By :user:`Deepak Cherian ` :issue:`1741`. * [v3] Reuse the downloaded array metadata when creating an ``Array``. - By :user:`Deepak Cherian `. + By :user:`Deepak Cherian ` :issue:`1734`. * Optimize ``Array.info`` so that it calls `getsize` only once. - By :user:`Deepak Cherian `. + By :user:`Deepak Cherian ` :issue:`1733`. * Override IPython ``_repr_*_`` methods to avoid expensive lookups against object stores. By :user:`Deepak Cherian ` :issue:`1716`. @@ -53,6 +54,21 @@ Enhancements * FSStore now raises rather than return bad data. By :user:`Martin Durant ` and :user:`Ian Carroll ` :issue:`1604`. +* Avoid redundant ``__contains__``. + By :user:`Deepak Cherian ` :issue:`1739`. + +Docs +~~~~ + +* Fix link to GCSMap in ``tutorial.rst``. + By :user:`Daniel Jahn ` :issue:`1689`. + +* Endorse `SPEC0000 `_ and state version support policy in ``installation.rst``. + By :user:`Sanket Verma ` :issue:`1665`. + +* Migrate v1 and v2 specification to `Zarr-Specs `_. + By :user:`Sanket Verma ` :issue:`1582`. + Maintenance ~~~~~~~~~~~ @@ -61,7 +77,12 @@ Maintenance * Bump minimum supported NumPy version to 1.23 (per spec 0000) By :user:`Joe Hamman ` :issue:`1719`. - + +* Minor fixes: Using ``is`` instead of ``type`` and removing unnecessary ``None``. + By :user:`Dimitri Papadopoulos Orfanos ` :issue:`1737`. + +* Fix tests failure related to Pytest 8. + By :user:`David Stansby ` :issue:`1714`. .. _release_2.17.1: From dd3dd96c7a5b7891bfa6be6ff9e989b49d00b0f5 Mon Sep 17 00:00:00 2001 From: Joe Hamman Date: Mon, 22 Apr 2024 12:45:07 -0700 Subject: [PATCH 079/130] Deprecate the experimental v3 implementation (#1802) * deprecate(exp-v3): Add a future warning about the pending removal of the experimental v3 implementation * ignore warning * add test --- pyproject.toml | 1 + zarr/_storage/store.py | 13 +++++++++++++ zarr/tests/test_storage_v3.py | 20 +++++++++++++++++++- 3 files changed, 33 insertions(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 0be79f990e..904c974424 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -137,6 +137,7 @@ filterwarnings = [ "error:::zarr.*", "ignore:PY_SSIZE_T_CLEAN will be required.*:DeprecationWarning", "ignore:The loop argument is deprecated since Python 3.8.*:DeprecationWarning", + "ignore:The experimental Zarr V3 implementation in this version .*:FutureWarning", ] diff --git a/zarr/_storage/store.py b/zarr/_storage/store.py index 0a08080548..69986ecadd 100644 --- a/zarr/_storage/store.py +++ b/zarr/_storage/store.py @@ -1,5 +1,6 @@ import abc import os +import warnings from collections import defaultdict from collections.abc import MutableMapping from copy import copy @@ -23,9 +24,21 @@ DEFAULT_ZARR_VERSION: ZARR_VERSION = 2 v3_api_available = os.environ.get("ZARR_V3_EXPERIMENTAL_API", "0").lower() not in ["0", "false"] +_has_warned_about_v3 = False # to avoid printing the warning multiple times def assert_zarr_v3_api_available(): + # we issue a warning about the experimental v3 implementation when it is first used + global _has_warned_about_v3 + if v3_api_available and not _has_warned_about_v3: + warnings.warn( + "The experimental Zarr V3 implementation in this version of Zarr-Python is not " + "in alignment with the final V3 specification. This version will be removed in " + "Zarr-Python 3 in favor of a spec compliant version.", + FutureWarning, + stacklevel=1, + ) + _has_warned_about_v3 = True if not v3_api_available: raise NotImplementedError( "# V3 reading and writing is experimental! To enable support, set:\n" diff --git a/zarr/tests/test_storage_v3.py b/zarr/tests/test_storage_v3.py index e15b2db743..c096f9cb02 100644 --- a/zarr/tests/test_storage_v3.py +++ b/zarr/tests/test_storage_v3.py @@ -4,12 +4,18 @@ import inspect import os import tempfile +import warnings import numpy as np import pytest import zarr -from zarr._storage.store import _get_hierarchy_metadata, v3_api_available, StorageTransformer +from zarr._storage.store import ( + _get_hierarchy_metadata, + assert_zarr_v3_api_available, + v3_api_available, + StorageTransformer, +) from zarr._storage.v3_storage_transformers import ShardingStorageTransformer, v3_sharding_available from zarr.core import Array from zarr.meta import _default_entry_point_metadata_v3 @@ -668,6 +674,18 @@ def test_top_level_imports(): assert not hasattr(zarr, store_name) # pragma: no cover +def test_assert_zarr_v3_api_available_warns_once(): + import zarr._storage.store + + zarr._storage.store._has_warned_about_v3 = False + warnings.resetwarnings() + with pytest.warns() as record: + assert_zarr_v3_api_available() + assert_zarr_v3_api_available() + assert len(record) == 1 + assert "The experimental Zarr V3 implementation" in str(record[0].message) + + def _get_public_and_dunder_methods(some_class): return set( name From 0a29fb3e40e57dd8de62f786a18c807982d5a2da Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 22 Apr 2024 20:26:11 +0000 Subject: [PATCH 080/130] chore: update pre-commit hooks (#1779) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit updates: - [github.com/astral-sh/ruff-pre-commit: v0.3.5 → v0.3.7](https://github.com/astral-sh/ruff-pre-commit/compare/v0.3.5...v0.3.7) - [github.com/psf/black: 24.3.0 → 24.4.0](https://github.com/psf/black/compare/24.3.0...24.4.0) - [github.com/pre-commit/pre-commit-hooks: v4.5.0 → v4.6.0](https://github.com/pre-commit/pre-commit-hooks/compare/v4.5.0...v4.6.0) Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- .pre-commit-config.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 24ff72a12f..0aa13b31a3 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -8,11 +8,11 @@ default_language_version: repos: - repo: https://github.com/astral-sh/ruff-pre-commit # Ruff version. - rev: 'v0.3.5' + rev: 'v0.4.1' hooks: - id: ruff - repo: https://github.com/psf/black - rev: 24.3.0 + rev: 24.4.0 hooks: - id: black - repo: https://github.com/codespell-project/codespell @@ -20,7 +20,7 @@ repos: hooks: - id: codespell - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.5.0 + rev: v4.6.0 hooks: - id: check-yaml - repo: https://github.com/pre-commit/mirrors-mypy From 9d046ea0d2878af7d15b3de3ec3036fe31661340 Mon Sep 17 00:00:00 2001 From: Deepak Cherian Date: Mon, 22 Apr 2024 14:34:17 -0600 Subject: [PATCH 081/130] Fix `is_total_slice` for size-1 dimensions (#1800) Closes #1730 Co-authored-by: Ryan Abernathey Co-authored-by: Joe Hamman --- docs/release.rst | 2 ++ zarr/tests/test_util.py | 9 +++++++++ zarr/util.py | 13 +++++++++++-- 3 files changed, 22 insertions(+), 2 deletions(-) diff --git a/docs/release.rst b/docs/release.rst index 3b5ab631df..07c2a47e7c 100644 --- a/docs/release.rst +++ b/docs/release.rst @@ -20,6 +20,8 @@ Unreleased Enhancements ~~~~~~~~~~~~ +* Performance improvement for reading and writing chunks if any of the dimensions is size 1. :issue:`1730` + By :user:`Deepak Cherian `. Docs diff --git a/zarr/tests/test_util.py b/zarr/tests/test_util.py index 1f7efc9214..d908c7b2d7 100644 --- a/zarr/tests/test_util.py +++ b/zarr/tests/test_util.py @@ -89,6 +89,15 @@ def test_is_total_slice(): assert not is_total_slice((slice(0, 50), slice(0, 50)), (100, 100)) assert not is_total_slice((slice(0, 100, 2), slice(0, 100)), (100, 100)) + # size-1 dimension edge-case + # https://github.com/zarr-developers/zarr-python/issues/1730 + assert is_total_slice((slice(0, 1),), (1,)) + # this is an equivalent selection (without a slice) + assert is_total_slice((0,), (1,)) + # same for multidimensional selection + assert is_total_slice((slice(0, 1), slice(0, 10)), (1, 10)) + assert is_total_slice((0, slice(0, 10)), (1, 10)) + with pytest.raises(TypeError): is_total_slice("foo", (100,)) diff --git a/zarr/util.py b/zarr/util.py index 848f1ed114..e58aed80ab 100644 --- a/zarr/util.py +++ b/zarr/util.py @@ -234,8 +234,17 @@ def is_total_slice(item, shape: Tuple[int]) -> bool: if isinstance(item, tuple): return all( ( - isinstance(it, slice) - and ((it == slice(None)) or ((it.stop - it.start == sh) and (it.step in [1, None]))) + ( + isinstance(it, slice) + and ( + (it == slice(None)) + or ((it.stop - it.start == sh) and (it.step in [1, None])) + ) + ) + # The only scalar edge case, indexing with int 0 along a size-1 dimension + # is identical to a total slice + # https://github.com/zarr-developers/zarr-python/issues/1730 + or (isinstance(it, int) and it == 0 and sh == 1) ) for it, sh in zip(item, shape) ) From 60b53b059e0d5b4fd57dab74520af0184da32006 Mon Sep 17 00:00:00 2001 From: Joe Hamman Date: Thu, 25 Apr 2024 06:14:40 -0700 Subject: [PATCH 082/130] add note to the top of the release page noting the plan for 2.18.* and 3.0 (#1816) --- docs/release.rst | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/docs/release.rst b/docs/release.rst index 07c2a47e7c..1c4e52f7c0 100644 --- a/docs/release.rst +++ b/docs/release.rst @@ -13,6 +13,11 @@ Release notes # to document your changes. On releases it will be # re-indented so that it does not show up in the notes. +.. note:: + Zarr-Python 2.18.* is expected be the final release in the 2.* series. Work on Zarr-Python 3.0 is underway. + See `GH1777 `_ for more details on the upcoming + 3.0 release. + .. _unreleased: Unreleased From f4f6e8692026201223c04850bbe13e0551471c39 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 29 Apr 2024 13:12:24 +0200 Subject: [PATCH 083/130] Bump conda-incubator/setup-miniconda from 3.0.3 to 3.0.4 (#1824) Bumps [conda-incubator/setup-miniconda](https://github.com/conda-incubator/setup-miniconda) from 3.0.3 to 3.0.4. - [Release notes](https://github.com/conda-incubator/setup-miniconda/releases) - [Changelog](https://github.com/conda-incubator/setup-miniconda/blob/main/CHANGELOG.md) - [Commits](https://github.com/conda-incubator/setup-miniconda/compare/v3.0.3...v3.0.4) --- updated-dependencies: - dependency-name: conda-incubator/setup-miniconda dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/minimal.yml | 2 +- .github/workflows/python-package.yml | 2 +- .github/workflows/windows-testing.yml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/minimal.yml b/.github/workflows/minimal.yml index dba6918514..b5b2f48d62 100644 --- a/.github/workflows/minimal.yml +++ b/.github/workflows/minimal.yml @@ -15,7 +15,7 @@ jobs: steps: - uses: actions/checkout@v4 - name: Setup Miniconda - uses: conda-incubator/setup-miniconda@v3.0.3 + uses: conda-incubator/setup-miniconda@v3.0.4 with: channels: conda-forge environment-file: environment.yml diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index 8ff6e9a2eb..f53cb2d9a9 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -44,7 +44,7 @@ jobs: with: fetch-depth: 0 - name: Setup Miniconda - uses: conda-incubator/setup-miniconda@v3.0.3 + uses: conda-incubator/setup-miniconda@v3.0.4 with: channels: conda-forge python-version: ${{ matrix.python-version }} diff --git a/.github/workflows/windows-testing.yml b/.github/workflows/windows-testing.yml index d580ef3f0e..ab86831aae 100644 --- a/.github/workflows/windows-testing.yml +++ b/.github/workflows/windows-testing.yml @@ -21,7 +21,7 @@ jobs: - uses: actions/checkout@v4 with: fetch-depth: 0 - - uses: conda-incubator/setup-miniconda@v3.0.3 + - uses: conda-incubator/setup-miniconda@v3.0.4 with: auto-update-conda: true python-version: ${{ matrix.python-version }} From 29b4acc45ed7b1f865c31bfa5d7beda4b3b9269a Mon Sep 17 00:00:00 2001 From: Joe Hamman Date: Tue, 30 Apr 2024 09:17:05 -0700 Subject: [PATCH 084/130] dep(docs): deprecate experimental v3 support in docs (#1807) * dep(docs): deprecate experimental v3 support in docs * Apply suggestions from code review Co-authored-by: Josh Moore Co-authored-by: Sanket Verma --------- Co-authored-by: Josh Moore Co-authored-by: Sanket Verma --- docs/api/v3.rst | 9 ++++----- docs/release.rst | 6 ++++++ zarr/convenience.py | 25 +++++++++++++++++++++++++ 3 files changed, 35 insertions(+), 5 deletions(-) diff --git a/docs/api/v3.rst b/docs/api/v3.rst index 7665b2ddd1..3503e3fe81 100644 --- a/docs/api/v3.rst +++ b/docs/api/v3.rst @@ -1,13 +1,12 @@ V3 Specification Implementation(``zarr._storage.v3``) ===================================================== -This module contains the implementation of the `Zarr V3 Specification `_. +This module contains an experimental implementation of the `Zarr V3 Specification `_. .. warning:: - Since Zarr Python 2.12 release, this module provides experimental infrastructure for reading and - writing the upcoming V3 spec of the Zarr format. Users wishing to prepare for the migration can set - the environment variable ``ZARR_V3_EXPERIMENTAL_API=1`` to begin experimenting, however data - written with this API should be expected to become stale, as the implementation will still change. + The experimental v3 implementation included in Zarr Python >2.12,<3 is not aligned with the final + V3 specification. This version is deprecated and will be removed in Zarr Python 3.0 in favor of a + spec compliant version. The new ``zarr._store.v3`` package has the necessary classes and functions for evaluating Zarr V3. Since the design is not finalised, the classes and functions are not automatically imported into diff --git a/docs/release.rst b/docs/release.rst index 1c4e52f7c0..811ede3d58 100644 --- a/docs/release.rst +++ b/docs/release.rst @@ -36,6 +36,12 @@ Docs Maintenance ~~~~~~~~~~~ +Deprecations +~~~~~~~~~~~~ + +* Deprecate experimental v3 support by issuing a `FutureWarning`. + Also updated docs to warn about using the experimental v3 version. + By :user:`Joe Hamman ` :issue:`1802` and :issue: `1807`. .. _release_2.17.2: diff --git a/zarr/convenience.py b/zarr/convenience.py index 7ca5d426f0..bd284e0844 100644 --- a/zarr/convenience.py +++ b/zarr/convenience.py @@ -55,6 +55,11 @@ def open(store: StoreLike = None, mode: str = "a", *, zarr_version=None, path=No The zarr protocol version to use. The default value of None will attempt to infer the version from `store` if possible, otherwise it will fall back to 2. + + .. warning:: `zarr_version=3` is currently using the experimental Zarr V3 + implementation. This implementation is not in sync with the final specification + and will be replaced with a spec compliant version in the version 3.0. + path : str or None, optional The path within the store to open. **kwargs @@ -150,6 +155,11 @@ def save_array(store: StoreLike, arr, *, zarr_version=None, path=None, **kwargs) The zarr protocol version to use when saving. The default value of None will attempt to infer the version from `store` if possible, otherwise it will fall back to 2. + + .. warning:: `zarr_version=3` is currently using the experimental Zarr V3 + implementation. This implementation is not in sync with the final specification + and will be replaced with a spec compliant version in the version 3.0. + path : str or None, optional The path within the store where the array will be saved. kwargs @@ -200,6 +210,11 @@ def save_group(store: StoreLike, *args, zarr_version=None, path=None, **kwargs): The zarr protocol version to use when saving. The default value of None will attempt to infer the version from `store` if possible, otherwise it will fall back to 2. + + .. warning:: `zarr_version=3` is currently using the experimental Zarr V3 + implementation. This implementation is not in sync with the final specification + and will be replaced with a spec compliant version in the version 3.0. + path : str or None, optional Path within the store where the group will be saved. kwargs @@ -282,6 +297,11 @@ def save(store: StoreLike, *args, zarr_version=None, path=None, **kwargs): The zarr protocol version to use when saving. The default value of None will attempt to infer the version from `store` if possible, otherwise it will fall back to 2. + + .. warning:: `zarr_version=3` is currently using the experimental Zarr V3 + implementation. This implementation is not in sync with the final specification + and will be replaced with a spec compliant version in the version 3.0. + path : str or None, optional The path within the group where the arrays will be saved. kwargs @@ -395,6 +415,11 @@ def load(store: StoreLike, zarr_version=None, path=None): The zarr protocol version to use when loading. The default value of None will attempt to infer the version from `store` if possible, otherwise it will fall back to 2. + + .. warning:: `zarr_version=3` is currently using the experimental Zarr V3 + implementation. This implementation is not in sync with the final specification + and will be replaced with a spec compliant version in the version 3.0. + path : str or None, optional The path within the store from which to load. From 7fa17b8e1350e8fd254ea530c7806b484fe1fc60 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 2 May 2024 00:26:02 +0000 Subject: [PATCH 085/130] Bump h5py from 3.10.0 to 3.11.0 (#1786) Bumps [h5py](https://github.com/h5py/h5py) from 3.10.0 to 3.11.0. - [Release notes](https://github.com/h5py/h5py/releases) - [Changelog](https://github.com/h5py/h5py/blob/master/docs/release_guide.rst) - [Commits](https://github.com/h5py/h5py/compare/3.10.0...3.11.0) --- updated-dependencies: - dependency-name: h5py dependency-type: direct:development update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Joe Hamman --- requirements_dev_optional.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements_dev_optional.txt b/requirements_dev_optional.txt index 809d1c0eee..951594460a 100644 --- a/requirements_dev_optional.txt +++ b/requirements_dev_optional.txt @@ -17,7 +17,7 @@ coverage pytest-cov==5.0.0 pytest-doctestplus==1.2.1 pytest-timeout==2.3.1 -h5py==3.10.0 +h5py==3.11.0 fsspec==2023.12.2 s3fs==2023.12.2 moto[server]>=5.0.1 From 9331430dfb1e348b8b2e5c6290cbce0956fd0cf1 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 2 May 2024 00:26:56 +0000 Subject: [PATCH 086/130] Bump redis from 5.0.3 to 5.0.4 (#1810) Bumps [redis](https://github.com/redis/redis-py) from 5.0.3 to 5.0.4. - [Release notes](https://github.com/redis/redis-py/releases) - [Changelog](https://github.com/redis/redis-py/blob/master/CHANGES) - [Commits](https://github.com/redis/redis-py/compare/v5.0.3...v5.0.4) --- updated-dependencies: - dependency-name: redis dependency-type: direct:development update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- requirements_dev_optional.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements_dev_optional.txt b/requirements_dev_optional.txt index 951594460a..3456cca21a 100644 --- a/requirements_dev_optional.txt +++ b/requirements_dev_optional.txt @@ -8,7 +8,7 @@ ipywidgets==8.1.2 # don't let pyup change pinning for azure-storage-blob, need to pin to older # version to get compatibility with azure storage emulator on appveyor (FIXME) azure-storage-blob==12.16.0 # pyup: ignore -redis==5.0.3 +redis==5.0.4 types-redis types-setuptools pymongo==4.6.3 From 2f2914456e391ec77032223e9587b1d3ef5608aa Mon Sep 17 00:00:00 2001 From: Joe Hamman Date: Mon, 6 May 2024 15:33:09 -0700 Subject: [PATCH 087/130] deprecate(stores): add deprecation warnings to stores that we plan to remove in v3 (#1801) * deprecate(stores): add deprecation warnings to DBMStore, LMDBStore, SQLiteStore, MongoDBStore, RedisStore, and ABSStore * filter warnings in pytest config * more deprecation warnings in docstrings * add release note --- docs/release.rst | 7 ++++ pyproject.toml | 1 + zarr/_storage/absstore.py | 15 ++++++++- zarr/_storage/store.py | 5 +++ zarr/storage.py | 68 +++++++++++++++++++++++++++++++++++++++ 5 files changed, 95 insertions(+), 1 deletion(-) diff --git a/docs/release.rst b/docs/release.rst index 811ede3d58..5184ab5f9f 100644 --- a/docs/release.rst +++ b/docs/release.rst @@ -43,6 +43,13 @@ Deprecations Also updated docs to warn about using the experimental v3 version. By :user:`Joe Hamman ` :issue:`1802` and :issue: `1807`. +Deprecations +~~~~~~~~~~~~ +* Deprecate the following stores: :class:`zarr.storage.DBMStore`, :class:`zarr.storage.LMDBStore`, + :class:`zarr.storage.SQLiteStore`, :class:`zarr.storage.MongoDBStore`, :class:`zarr.storage.RedisStore`, + and :class:`zarr.storage.ABSStore`. These stores are slated to be removed from Zarr-Python in version 3.0. + By :user:`Joe Hamman ` :issue:`1801`. + .. _release_2.17.2: 2.17.2 diff --git a/pyproject.toml b/pyproject.toml index 904c974424..f2356480bd 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -137,6 +137,7 @@ filterwarnings = [ "error:::zarr.*", "ignore:PY_SSIZE_T_CLEAN will be required.*:DeprecationWarning", "ignore:The loop argument is deprecated since Python 3.8.*:DeprecationWarning", + "ignore:The .* is deprecated and will be removed in a Zarr-Python version 3*:FutureWarning", "ignore:The experimental Zarr V3 implementation in this version .*:FutureWarning", ] diff --git a/zarr/_storage/absstore.py b/zarr/_storage/absstore.py index 217b2a29e0..5d2606f2f2 100644 --- a/zarr/_storage/absstore.py +++ b/zarr/_storage/absstore.py @@ -5,7 +5,14 @@ from numcodecs.compat import ensure_bytes from zarr.util import normalize_storage_path -from zarr._storage.store import _get_metadata_suffix, data_root, meta_root, Store, StoreV3 +from zarr._storage.store import ( + _get_metadata_suffix, + data_root, + meta_root, + Store, + StoreV3, + V3_DEPRECATION_MESSAGE, +) from zarr.types import DIMENSION_SEPARATOR __doctest_requires__ = { @@ -73,6 +80,12 @@ def __init__( dimension_separator: Optional[DIMENSION_SEPARATOR] = None, client=None, ): + warnings.warn( + V3_DEPRECATION_MESSAGE.format(store=self.__class__.__name__), + FutureWarning, + stacklevel=3, + ) + self._dimension_separator = dimension_separator self.prefix = normalize_storage_path(prefix) if client is None: diff --git a/zarr/_storage/store.py b/zarr/_storage/store.py index 69986ecadd..dba29d13c0 100644 --- a/zarr/_storage/store.py +++ b/zarr/_storage/store.py @@ -26,6 +26,11 @@ v3_api_available = os.environ.get("ZARR_V3_EXPERIMENTAL_API", "0").lower() not in ["0", "false"] _has_warned_about_v3 = False # to avoid printing the warning multiple times +V3_DEPRECATION_MESSAGE = ( + "The {store} is deprecated and will be removed in a Zarr-Python version 3, see " + "https://github.com/zarr-developers/zarr-python/issues/1274 for more information." +) + def assert_zarr_v3_api_available(): # we issue a warning about the experimental v3 implementation when it is first used diff --git a/zarr/storage.py b/zarr/storage.py index 10f55f0ba3..772fa7646a 100644 --- a/zarr/storage.py +++ b/zarr/storage.py @@ -88,6 +88,7 @@ DEFAULT_ZARR_VERSION, BaseStore, Store, + V3_DEPRECATION_MESSAGE, ) __doctest_requires__ = { @@ -1604,6 +1605,12 @@ class NestedDirectoryStore(DirectoryStore): special handling for chunk keys so that chunk files for multidimensional arrays are stored in a nested directory tree. + .. deprecated:: 2.18.0 + NestedDirectoryStore will be removed in Zarr-Python 3.0 where controlling + the chunk key encoding will be supported as part of the array metadata. See + `GH1274 `_ + for more information. + Parameters ---------- path : string @@ -1675,6 +1682,13 @@ class NestedDirectoryStore(DirectoryStore): def __init__( self, path, normalize_keys=False, dimension_separator: Optional[DIMENSION_SEPARATOR] = "/" ): + + warnings.warn( + V3_DEPRECATION_MESSAGE.format(store=self.__class__.__name__), + FutureWarning, + stacklevel=2, + ) + super().__init__(path, normalize_keys=normalize_keys) if dimension_separator is None: dimension_separator = "/" @@ -1995,6 +2009,11 @@ def migrate_1to2(store): class DBMStore(Store): """Storage class using a DBM-style database. + .. deprecated:: 2.18.0 + DBMStore will be removed in Zarr-Python 3.0. See + `GH1274 `_ + for more information. + Parameters ---------- path : string @@ -2083,6 +2102,12 @@ def __init__( dimension_separator: Optional[DIMENSION_SEPARATOR] = None, **open_kwargs, ): + warnings.warn( + V3_DEPRECATION_MESSAGE.format(store=self.__class__.__name__), + FutureWarning, + stacklevel=2, + ) + if open is None: import dbm @@ -2200,6 +2225,10 @@ class LMDBStore(Store): """Storage class using LMDB. Requires the `lmdb `_ package to be installed. + .. deprecated:: 2.18.0 + LMDBStore will be removed in Zarr-Python 3.0. See + `GH1274 `_ + for more information. Parameters ---------- @@ -2261,6 +2290,12 @@ def __init__( ): import lmdb + warnings.warn( + V3_DEPRECATION_MESSAGE.format(store=self.__class__.__name__), + FutureWarning, + stacklevel=2, + ) + # set default memory map size to something larger than the lmdb default, which is # very likely to be too small for any moderate array (logic copied from zict) map_size = 2**40 if sys.maxsize >= 2**32 else 2**28 @@ -2580,6 +2615,11 @@ def __delitem__(self, key): class SQLiteStore(Store): """Storage class using SQLite. + .. deprecated:: 2.18.0 + SQLiteStore will be removed in Zarr-Python 3.0. See + `GH1274 `_ + for more information. + Parameters ---------- path : string @@ -2612,6 +2652,12 @@ class SQLiteStore(Store): def __init__(self, path, dimension_separator: Optional[DIMENSION_SEPARATOR] = None, **kwargs): import sqlite3 + warnings.warn( + V3_DEPRECATION_MESSAGE.format(store=self.__class__.__name__), + FutureWarning, + stacklevel=2, + ) + self._dimension_separator = dimension_separator # normalize path @@ -2778,6 +2824,11 @@ class MongoDBStore(Store): .. note:: This is an experimental feature. + .. deprecated:: 2.18.0 + MongoDBStore will be removed in Zarr-Python 3.0. See + `GH1274 `_ + for more information. + Requires the `pymongo `_ package to be installed. @@ -2810,6 +2861,12 @@ def __init__( ): import pymongo + warnings.warn( + V3_DEPRECATION_MESSAGE.format(store=self.__class__.__name__), + FutureWarning, + stacklevel=2, + ) + self._database = database self._collection = collection self._dimension_separator = dimension_separator @@ -2866,6 +2923,11 @@ class RedisStore(Store): .. note:: This is an experimental feature. + .. deprecated:: 2.18.0 + RedisStore will be removed in Zarr-Python 3.0. See + `GH1274 `_ + for more information. + Requires the `redis `_ package to be installed. @@ -2885,6 +2947,12 @@ def __init__( ): import redis + warnings.warn( + V3_DEPRECATION_MESSAGE.format(store=self.__class__.__name__), + FutureWarning, + stacklevel=2, + ) + self._prefix = prefix self._kwargs = kwargs self._dimension_separator = dimension_separator From 863c3c0838adc1f65eb5fcddb6cf8fd65480acb5 Mon Sep 17 00:00:00 2001 From: Joe Hamman Date: Mon, 6 May 2024 21:56:37 -0700 Subject: [PATCH 088/130] use np.inf instead of PINF/NINF (#1842) * use np.inf instead of PINF/NINF * update release notes --- docs/release.rst | 12 +++++------- zarr/meta.py | 4 ++-- zarr/tests/test_meta.py | 4 ++-- 3 files changed, 9 insertions(+), 11 deletions(-) diff --git a/docs/release.rst b/docs/release.rst index 5184ab5f9f..0cb1777210 100644 --- a/docs/release.rst +++ b/docs/release.rst @@ -25,9 +25,8 @@ Unreleased Enhancements ~~~~~~~~~~~~ -* Performance improvement for reading and writing chunks if any of the dimensions is size 1. :issue:`1730` - By :user:`Deepak Cherian `. - +* Performance improvement for reading and writing chunks if any of the dimensions is size 1. + By :user:`Deepak Cherian ` :issue:`1730`. Docs ~~~~ @@ -35,16 +34,15 @@ Docs Maintenance ~~~~~~~~~~~ +* Minor updates to use `np.inf` instead of `np.PINF` / `np.NINF` in preparation for NumPy 2.0.0 release. + By :user:`Joe Hamman ` :issue:`1842`. Deprecations ~~~~~~~~~~~~ * Deprecate experimental v3 support by issuing a `FutureWarning`. Also updated docs to warn about using the experimental v3 version. - By :user:`Joe Hamman ` :issue:`1802` and :issue: `1807`. - -Deprecations -~~~~~~~~~~~~ + By :user:`Joe Hamman ` :issue:`1802` and :issue:`1807`. * Deprecate the following stores: :class:`zarr.storage.DBMStore`, :class:`zarr.storage.LMDBStore`, :class:`zarr.storage.SQLiteStore`, :class:`zarr.storage.MongoDBStore`, :class:`zarr.storage.RedisStore`, and :class:`zarr.storage.ABSStore`. These stores are slated to be removed from Zarr-Python in version 3.0. diff --git a/zarr/meta.py b/zarr/meta.py index 4b360270de..747d8bec8a 100644 --- a/zarr/meta.py +++ b/zarr/meta.py @@ -227,9 +227,9 @@ def decode_fill_value(cls, v: Any, dtype: np.dtype, object_codec: Any = None) -> if v == "NaN": return np.nan elif v == "Infinity": - return np.PINF + return np.inf elif v == "-Infinity": - return np.NINF + return -np.inf else: return np.array(v, dtype=dtype)[()] elif dtype.kind in "c": diff --git a/zarr/tests/test_meta.py b/zarr/tests/test_meta.py index c85d3f923f..54347835d7 100644 --- a/zarr/tests/test_meta.py +++ b/zarr/tests/test_meta.py @@ -382,8 +382,8 @@ def test_encode_decode_array_structured(): def test_encode_decode_fill_values_nan(): fills = ( (np.nan, "NaN", np.isnan), - (np.NINF, "-Infinity", np.isneginf), - (np.PINF, "Infinity", np.isposinf), + (-np.inf, "-Infinity", np.isneginf), + (np.inf, "Infinity", np.isposinf), ) for v, s, f in fills: From 360eb53f636c33bf255682f53078e91bf6e24094 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 6 May 2024 22:40:05 -0700 Subject: [PATCH 089/130] chore: update pre-commit hooks (#1825) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit updates: - [github.com/astral-sh/ruff-pre-commit: v0.4.1 → v0.4.3](https://github.com/astral-sh/ruff-pre-commit/compare/v0.4.1...v0.4.3) - [github.com/psf/black: 24.4.0 → 24.4.2](https://github.com/psf/black/compare/24.4.0...24.4.2) - [github.com/pre-commit/mirrors-mypy: v1.9.0 → v1.10.0](https://github.com/pre-commit/mirrors-mypy/compare/v1.9.0...v1.10.0) Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- .pre-commit-config.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 0aa13b31a3..747cb86688 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -8,11 +8,11 @@ default_language_version: repos: - repo: https://github.com/astral-sh/ruff-pre-commit # Ruff version. - rev: 'v0.4.1' + rev: 'v0.4.3' hooks: - id: ruff - repo: https://github.com/psf/black - rev: 24.4.0 + rev: 24.4.2 hooks: - id: black - repo: https://github.com/codespell-project/codespell @@ -24,7 +24,7 @@ repos: hooks: - id: check-yaml - repo: https://github.com/pre-commit/mirrors-mypy - rev: v1.9.0 + rev: v1.10.0 hooks: - id: mypy files: zarr From 270aff18dc780a0a9e7409455277aae017190765 Mon Sep 17 00:00:00 2001 From: Dimitri Papadopoulos Orfanos <3234522+DimitriPapadopoulos@users.noreply.github.com> Date: Tue, 7 May 2024 17:29:34 +0200 Subject: [PATCH 090/130] Enable ruff/bugbear rules (B) and fix issues (#1702) * Enable ruff/bugbear rules (B) As suggested by Repo-Review. * Fix ruff/bugbear issue (B007) B007 Loop control variable `key` not used within loop body https://docs.astral.sh/ruff/rules/unused-loop-control-variable/ * Fix ruff/bugbear issue (B015) B015 Pointless comparison. Did you mean to assign a value? Otherwise, prepend `assert` or remove it. https://docs.astral.sh/ruff/rules/useless-comparison/ * Fix ruff/bugbear issues (B028) B028 No explicit `stacklevel` keyword argument found https://docs.astral.sh/ruff/rules/no-explicit-stacklevel/ * Fix ruff/bugbear issues (B904) B904 Within an `except` clause, raise exceptions with `raise ... from err` or `raise ... from None` to distinguish them from errors in exception handling https://docs.astral.sh/ruff/rules/raise-without-from-inside-except/ * Document changes in docs/release.rst * Disable ruff/bugbear rule (B017) B017 `pytest.raises(Exception)` should be considered evil https://docs.astral.sh/ruff/rules/assert-raises-exception/ --------- Co-authored-by: Joe Hamman --- docs/release.rst | 3 +++ pyproject.toml | 5 +++++ zarr/_storage/absstore.py | 8 ++++---- zarr/_storage/v3_storage_transformers.py | 8 ++++---- zarr/core.py | 4 ++-- zarr/creation.py | 2 +- zarr/hierarchy.py | 10 +++++----- zarr/indexing.py | 2 +- zarr/meta.py | 6 +++--- zarr/meta_v1.py | 2 +- zarr/n5.py | 22 +++++++++++++++++----- zarr/storage.py | 16 ++++++++++------ zarr/tests/test_meta.py | 2 +- zarr/tests/test_storage.py | 6 +++--- zarr/tests/test_storage_v3.py | 2 +- zarr/util.py | 12 ++++++------ 16 files changed, 67 insertions(+), 43 deletions(-) diff --git a/docs/release.rst b/docs/release.rst index 0cb1777210..e2bc40bf99 100644 --- a/docs/release.rst +++ b/docs/release.rst @@ -34,6 +34,9 @@ Docs Maintenance ~~~~~~~~~~~ +* Enable ruff/bugbear rules (B) and fix issues. + By :user:`Dimitri Papadopoulos Orfanos ` :issue:`1702`. + * Minor updates to use `np.inf` instead of `np.PINF` / `np.NINF` in preparation for NumPy 2.0.0 release. By :user:`Joe Hamman ` :issue:`1842`. diff --git a/pyproject.toml b/pyproject.toml index f2356480bd..dacd45ec2c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -103,6 +103,11 @@ exclude = [ "docs" ] +[tool.ruff.lint] +extend-select = [ + "B" +] + [tool.black] line-length = 100 exclude = ''' diff --git a/zarr/_storage/absstore.py b/zarr/_storage/absstore.py index 5d2606f2f2..1e49754f38 100644 --- a/zarr/_storage/absstore.py +++ b/zarr/_storage/absstore.py @@ -156,8 +156,8 @@ def __getitem__(self, key): blob_name = self._append_path_to_prefix(key) try: return self.client.download_blob(blob_name).readall() - except ResourceNotFoundError: - raise KeyError(f"Blob {blob_name} not found") + except ResourceNotFoundError as e: + raise KeyError(f"Blob {blob_name} not found") from e def __setitem__(self, key, value): value = ensure_bytes(value) @@ -169,8 +169,8 @@ def __delitem__(self, key): try: self.client.delete_blob(self._append_path_to_prefix(key)) - except ResourceNotFoundError: - raise KeyError(f"Blob {key} not found") + except ResourceNotFoundError as e: + raise KeyError(f"Blob {key} not found") from e def __eq__(self, other): return ( diff --git a/zarr/_storage/v3_storage_transformers.py b/zarr/_storage/v3_storage_transformers.py index 37e56f8ecd..00467d44f9 100644 --- a/zarr/_storage/v3_storage_transformers.py +++ b/zarr/_storage/v3_storage_transformers.py @@ -183,8 +183,8 @@ def __getitem__(self, key): shard_key, chunk_subkey = self._key_to_shard(key) try: full_shard_value = self.inner_store[shard_key] - except KeyError: - raise KeyError(key) + except KeyError as e: + raise KeyError(key) from e index = self._get_index_from_buffer(full_shard_value) chunk_slice = index.get_chunk_slice(chunk_subkey) if chunk_slice is not None: @@ -265,8 +265,8 @@ def __delitem__(self, key): shard_key, chunk_subkey = self._key_to_shard(key) try: index = self._get_index_from_store(shard_key) - except KeyError: - raise KeyError(key) + except KeyError as e: + raise KeyError(key) from e index.set_chunk_slice(chunk_subkey, None) diff --git a/zarr/core.py b/zarr/core.py index 1bd081acee..6aa86b6465 100644 --- a/zarr/core.py +++ b/zarr/core.py @@ -200,8 +200,8 @@ def _load_metadata_nosync(self): try: mkey = _prefix_to_array_key(self._store, self._key_prefix) meta_bytes = self._store[mkey] - except KeyError: - raise ArrayNotFoundError(self._path) + except KeyError as e: + raise ArrayNotFoundError(self._path) from e else: # decode and store metadata as instance members meta = self._store._metadata_class.decode_array_metadata(meta_bytes) diff --git a/zarr/creation.py b/zarr/creation.py index c541531d54..9b2b1d6d4c 100644 --- a/zarr/creation.py +++ b/zarr/creation.py @@ -297,7 +297,7 @@ def _kwargs_compat(compressor, fill_value, kwargs): # ignore other keyword arguments for k in kwargs: - warn(f"ignoring keyword argument {k!r}") + warn(f"ignoring keyword argument {k!r}", stacklevel=2) return compressor, fill_value diff --git a/zarr/hierarchy.py b/zarr/hierarchy.py index 0fb07dd620..8894a5ed57 100644 --- a/zarr/hierarchy.py +++ b/zarr/hierarchy.py @@ -187,16 +187,16 @@ def __init__( mkey = _prefix_to_group_key(self._store, self._key_prefix) assert not mkey.endswith("root/.group") meta_bytes = store[mkey] - except KeyError: + except KeyError as e: if self._version == 2: - raise GroupNotFoundError(path) + raise GroupNotFoundError(path) from e else: implicit_prefix = meta_root + self._key_prefix if self._store.list_prefix(implicit_prefix): # implicit group does not have any metadata self._meta = None else: - raise GroupNotFoundError(path) + raise GroupNotFoundError(path) from e else: self._meta = self._store._metadata_class.decode_group_metadata(meta_bytes) @@ -536,8 +536,8 @@ def __getattr__(self, item): # allow access to group members via dot notation try: return self.__getitem__(item) - except KeyError: - raise AttributeError + except KeyError as e: + raise AttributeError from e def __dir__(self): # noinspection PyUnresolvedReferences diff --git a/zarr/indexing.py b/zarr/indexing.py index 9889fcadad..2f2402fe27 100644 --- a/zarr/indexing.py +++ b/zarr/indexing.py @@ -932,7 +932,7 @@ def check_fields(fields, dtype): # multiple field selection out_dtype = np.dtype([(f, dtype[f]) for f in fields]) except KeyError as e: - raise IndexError(f"invalid 'fields' argument, field not found: {e!r}") + raise IndexError(f"invalid 'fields' argument, field not found: {e!r}") from e else: return out_dtype else: diff --git a/zarr/meta.py b/zarr/meta.py index 747d8bec8a..5430ab305d 100644 --- a/zarr/meta.py +++ b/zarr/meta.py @@ -310,8 +310,8 @@ def decode_dtype(cls, d, validate=True): # extract the type from the extension info try: d = d["type"] - except KeyError: - raise KeyError("Extended dtype info must provide a key named 'type'.") + except KeyError as e: + raise KeyError("Extended dtype info must provide a key named 'type'.") from e d = cls._decode_dtype_descr(d) dtype = np.dtype(d) if validate: @@ -518,7 +518,7 @@ def decode_array_metadata(cls, s: Union[MappingType, bytes, str]) -> MappingType meta["storage_transformers"] = storage_transformers except Exception as e: - raise MetadataError(f"error decoding metadata: {e}") + raise MetadataError(f"error decoding metadata: {e}") from e else: return meta diff --git a/zarr/meta_v1.py b/zarr/meta_v1.py index 65bfd3488e..714f55f477 100644 --- a/zarr/meta_v1.py +++ b/zarr/meta_v1.py @@ -23,7 +23,7 @@ def decode_metadata(b): order=meta["order"], ) except Exception as e: - raise MetadataError(f"error decoding metadata: {e}") + raise MetadataError(f"error decoding metadata: {e}") from e else: return meta diff --git a/zarr/n5.py b/zarr/n5.py index fdd3d5babf..3d3e9afa26 100644 --- a/zarr/n5.py +++ b/zarr/n5.py @@ -125,7 +125,11 @@ def __setitem__(self, key: str, value: Any): for k in n5_keywords: if k in zarr_attrs: - warnings.warn(f"Attribute {k} is a reserved N5 keyword", UserWarning) + warnings.warn( + f"Attribute {k} is a reserved N5 keyword", + UserWarning, + stacklevel=2, + ) # remove previous user attributes for k in list(n5_attrs.keys()): @@ -327,7 +331,10 @@ class N5FSStore(FSStore): def __init__(self, *args, **kwargs): if "dimension_separator" in kwargs: kwargs.pop("dimension_separator") - warnings.warn("Keyword argument `dimension_separator` will be ignored") + warnings.warn( + "Keyword argument `dimension_separator` will be ignored", + stacklevel=2, + ) dimension_separator = "." super().__init__(*args, dimension_separator=dimension_separator, **kwargs) @@ -411,7 +418,11 @@ def __setitem__(self, key: str, value: Any): for k in n5_keywords: if k in zarr_attrs.keys(): - warnings.warn(f"Attribute {k} is a reserved N5 keyword", UserWarning) + warnings.warn( + f"Attribute {k} is a reserved N5 keyword", + UserWarning, + stacklevel=2, + ) # replace previous user attributes for k in list(n5_attrs.keys()): @@ -597,8 +608,8 @@ def array_metadata_to_n5(array_metadata: Dict[str, Any], top_level=False) -> Dic array_metadata["n5"] = N5_FORMAT try: dtype = np.dtype(array_metadata["dataType"]) - except TypeError: - raise TypeError(f"Data type {array_metadata['dataType']} is not supported by N5") + except TypeError as e: + raise TypeError(f"Data type {array_metadata['dataType']} is not supported by N5") from e array_metadata["dataType"] = dtype.name array_metadata["dimensions"] = array_metadata["dimensions"][::-1] @@ -711,6 +722,7 @@ def compressor_config_to_n5(compressor_config: Optional[Dict[str, Any]]) -> Dict "Not all N5 implementations support lzma compression (yet). You " "might not be able to open the dataset with another N5 library.", RuntimeWarning, + stacklevel=2, ) n5_config["format"] = _compressor_config["format"] n5_config["check"] = _compressor_config["check"] diff --git a/zarr/storage.py b/zarr/storage.py index 772fa7646a..f412870f75 100644 --- a/zarr/storage.py +++ b/zarr/storage.py @@ -589,11 +589,15 @@ def _init_array_metadata( "missing object_codec for object array; this will raise a " "ValueError in version 3.0", FutureWarning, + stacklevel=2, ) else: filters_config.insert(0, object_codec.get_config()) elif object_codec is not None: - warnings.warn("an object_codec is only needed for object arrays") + warnings.warn( + "an object_codec is only needed for object arrays", + stacklevel=2, + ) # use null to indicate no filters if not filters_config: @@ -869,8 +873,8 @@ def __getitem__(self, item: str): parent, key = self._get_parent(item) try: value = parent[key] - except KeyError: - raise KeyError(item) + except KeyError as e: + raise KeyError(item) from e else: if isinstance(value, self.cls): raise KeyError(item) @@ -888,8 +892,8 @@ def __delitem__(self, item: str): parent, key = self._get_parent(item) try: del parent[key] - except KeyError: - raise KeyError(item) + except KeyError as e: + raise KeyError(item) from e def __contains__(self, item: str): # type: ignore[override] try: @@ -1137,7 +1141,7 @@ def __setitem__(self, key, value): os.makedirs(dir_path) except OSError as e: if e.errno != errno.EEXIST: - raise KeyError(key) + raise KeyError(key) from e # write to temporary file # note we're not using tempfile.NamedTemporaryFile to avoid restrictive file permissions diff --git a/zarr/tests/test_meta.py b/zarr/tests/test_meta.py index 54347835d7..f9010d6788 100644 --- a/zarr/tests/test_meta.py +++ b/zarr/tests/test_meta.py @@ -601,7 +601,7 @@ def test_metadata3_exceptions(): required = ["zarr_format", "metadata_encoding", "metadata_key_suffix", "extensions"] for key in required: meta = copy.copy(_default_entry_point_metadata_v3) - meta.pop("zarr_format") + meta.pop(key) with pytest.raises(ValueError): # cannot encode metadata that is missing a required key Metadata3.encode_hierarchy_metadata(meta) diff --git a/zarr/tests/test_storage.py b/zarr/tests/test_storage.py index ae8a56fa61..da690f5959 100644 --- a/zarr/tests/test_storage.py +++ b/zarr/tests/test_storage.py @@ -1353,13 +1353,13 @@ def test_exceptions(self, memory_store): # no exception from FSStore.getitems getting KeyError assert group.store.getitems(["foo"], contexts={}) == {} # exception from FSStore.getitems getting AttributeError - with pytest.raises(Exception): + with pytest.raises(Exception): # noqa: B017 group.store.getitems(["x/0"], contexts={}) # exception from FSStore.getitems getting AttributeError - with pytest.raises(Exception): + with pytest.raises(Exception): # noqa: B017 x[...] # exception from FSStore.__getitem__ getting AttributeError - with pytest.raises(Exception): + with pytest.raises(Exception): # noqa: B017 y[...] diff --git a/zarr/tests/test_storage_v3.py b/zarr/tests/test_storage_v3.py index c096f9cb02..e8675786e0 100644 --- a/zarr/tests/test_storage_v3.py +++ b/zarr/tests/test_storage_v3.py @@ -286,7 +286,7 @@ def test_rename_nonexisting(self): def test_get_partial_values(self): store = self.create_store() - store.supports_efficient_get_partial_values in [True, False] + assert store.supports_efficient_get_partial_values in [True, False] store[data_root + "foo"] = b"abcdefg" store[data_root + "baz"] = b"z" assert [b"a"] == store.get_partial_values([(data_root + "foo", (0, 1))]) diff --git a/zarr/util.py b/zarr/util.py index e58aed80ab..8a96f92c24 100644 --- a/zarr/util.py +++ b/zarr/util.py @@ -198,11 +198,11 @@ def normalize_dtype(dtype: Union[str, np.dtype], object_codec) -> Tuple[np.dtype args = [] try: object_codec = codec_registry[codec_id](*args) - except KeyError: # pragma: no cover + except KeyError as e: # pragma: no cover raise ValueError( f"codec {codec_id!r} for object type {key!r} is not " f"available; please provide an object_codec manually" - ) + ) from e return dtype, object_codec dtype = np.dtype(dtype) @@ -332,7 +332,7 @@ def normalize_fill_value(fill_value, dtype: np.dtype): raise ValueError( f"fill_value {fill_value!r} is not valid for dtype {dtype}; " f"nested exception: {e}" - ) + ) from e return fill_value @@ -492,13 +492,13 @@ def tree_widget_sublist(node, root=False, expand=False): def tree_widget(group, expand, level): try: import ipytree - except ImportError as error: + except ImportError as e: raise ImportError( - f"{error}: Run `pip install zarr[jupyter]` or `conda install ipytree`" + f"{e}: Run `pip install zarr[jupyter]` or `conda install ipytree`" f"to get the required ipytree dependency for displaying the tree " f"widget. If using jupyterlab<3, you also need to run " f"`jupyter labextension install ipytree`" - ) + ) from e result = ipytree.Tree() root = TreeNode(group, level=level) From 056657ca5ed70aa3d77a9e2db42253fca39800b0 Mon Sep 17 00:00:00 2001 From: Joe Hamman Date: Tue, 7 May 2024 14:01:44 -0700 Subject: [PATCH 091/130] Release notes for 2.18.0 (#1843) * doc: cleanup release notes for 2.18.0 --- docs/release.rst | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/docs/release.rst b/docs/release.rst index e2bc40bf99..ba26549402 100644 --- a/docs/release.rst +++ b/docs/release.rst @@ -18,20 +18,16 @@ Release notes See `GH1777 `_ for more details on the upcoming 3.0 release. -.. _unreleased: +.. _release_2.18.0: -Unreleased ----------- +2.18.0 +------ Enhancements ~~~~~~~~~~~~ * Performance improvement for reading and writing chunks if any of the dimensions is size 1. By :user:`Deepak Cherian ` :issue:`1730`. -Docs -~~~~ - - Maintenance ~~~~~~~~~~~ * Enable ruff/bugbear rules (B) and fix issues. @@ -46,6 +42,7 @@ Deprecations * Deprecate experimental v3 support by issuing a `FutureWarning`. Also updated docs to warn about using the experimental v3 version. By :user:`Joe Hamman ` :issue:`1802` and :issue:`1807`. + * Deprecate the following stores: :class:`zarr.storage.DBMStore`, :class:`zarr.storage.LMDBStore`, :class:`zarr.storage.SQLiteStore`, :class:`zarr.storage.MongoDBStore`, :class:`zarr.storage.RedisStore`, and :class:`zarr.storage.ABSStore`. These stores are slated to be removed from Zarr-Python in version 3.0. From cb4230dd9a2a0c68b78384b24a2a5567df0b704a Mon Sep 17 00:00:00 2001 From: Joe Hamman Date: Tue, 7 May 2024 21:27:39 -0700 Subject: [PATCH 092/130] Update release.rst (#1850) --- docs/release.rst | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/docs/release.rst b/docs/release.rst index ba26549402..e2f9f3de85 100644 --- a/docs/release.rst +++ b/docs/release.rst @@ -18,6 +18,23 @@ Release notes See `GH1777 `_ for more details on the upcoming 3.0 release. +.. _unreleased: + +Unreleased +---------- + +Enhancements +~~~~~~~~~~~~ + +Docs +~~~~ + +Maintenance +~~~~~~~~~~~ + +Deprecations +~~~~~~~~~~~~ + .. _release_2.18.0: 2.18.0 From 8979205fbebd991974172ac23b436372911f4666 Mon Sep 17 00:00:00 2001 From: Joe Hamman Date: Mon, 13 May 2024 16:45:41 -0700 Subject: [PATCH 093/130] Group dependabot updates (#1854) --- .github/dependabot.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/dependabot.yml b/.github/dependabot.yml index d8e8d4d57a..5a0befe9b5 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -5,6 +5,10 @@ updates: directory: "/" schedule: interval: "daily" + groups: + requirements: + patterns: + - "*" - package-ecosystem: "github-actions" directory: "/" schedule: From 8264acebc5e1671c36bc434f14cac914f5a099e1 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 13 May 2024 20:33:05 -0700 Subject: [PATCH 094/130] chore: update pre-commit hooks (#1876) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit updates: - [github.com/astral-sh/ruff-pre-commit: v0.4.3 → v0.4.4](https://github.com/astral-sh/ruff-pre-commit/compare/v0.4.3...v0.4.4) Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 747cb86688..be57770200 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -8,7 +8,7 @@ default_language_version: repos: - repo: https://github.com/astral-sh/ruff-pre-commit # Ruff version. - rev: 'v0.4.3' + rev: 'v0.4.4' hooks: - id: ruff - repo: https://github.com/psf/black From 908e920bfd52371a4148b19d305262a804bc0201 Mon Sep 17 00:00:00 2001 From: Deepak Cherian Date: Thu, 16 May 2024 16:37:54 -0600 Subject: [PATCH 095/130] Fix a regression with scalar indexing due to #1800 (#1875) --- docs/release.rst | 2 ++ zarr/core.py | 4 +++- zarr/indexing.py | 2 ++ zarr/tests/test_core.py | 49 +++++++++++++++++++++++++++++++++++++++++ 4 files changed, 56 insertions(+), 1 deletion(-) diff --git a/docs/release.rst b/docs/release.rst index e2f9f3de85..a81be4638f 100644 --- a/docs/release.rst +++ b/docs/release.rst @@ -31,6 +31,8 @@ Docs Maintenance ~~~~~~~~~~~ +* Fix a regression when getting or setting a single value from arrays with size-1 chunks. + By :user:`Deepak Cherian ` :issue:`1874` Deprecations ~~~~~~~~~~~~ diff --git a/zarr/core.py b/zarr/core.py index 6aa86b6465..b1ccd203db 100644 --- a/zarr/core.py +++ b/zarr/core.py @@ -2030,7 +2030,9 @@ def _process_chunk( and not self._filters and self._dtype != object ): - dest = out[out_selection] + # For 0D arrays out_selection = () and out[out_selection] is a scalar + # Avoid that + dest = out[out_selection] if out_selection else out # Assume that array-like objects that doesn't have a # `writeable` flag is writable. dest_is_writable = getattr(dest, "writeable", True) diff --git a/zarr/indexing.py b/zarr/indexing.py index 2f2402fe27..35c1e813b1 100644 --- a/zarr/indexing.py +++ b/zarr/indexing.py @@ -52,6 +52,8 @@ def is_scalar(value, dtype): return True if isinstance(value, tuple) and dtype.names and len(value) == len(dtype.names): return True + if dtype.kind == "O" and not isinstance(value, np.ndarray): + return True return False diff --git a/zarr/tests/test_core.py b/zarr/tests/test_core.py index 730f724314..01a78ecd68 100644 --- a/zarr/tests/test_core.py +++ b/zarr/tests/test_core.py @@ -3157,3 +3157,52 @@ def test_issue_1279(tmpdir): written_data = ds_reopened[:] assert_array_equal(data, written_data) + + +def test_scalar_indexing(): + store = zarr.KVStore({}) + + store["a"] = zarr.create((3,), chunks=(1,), store=store) + store["a"][:] = [1, 2, 3] + + assert store["a"][1] == np.array(2.0) + assert store["a"][(1,)] == np.array(2.0) + + store["a"][slice(1)] = [-1] + assert store["a"][0] == np.array(-1) + + store["a"][0] = -2 + assert store["a"][0] == np.array(-2) + + store["a"][slice(1)] = (-3,) + assert store["a"][0] == np.array(-3) + + +def test_object_array_indexing(): + # regression test for #1874 + from numcodecs import MsgPack + + root = zarr.group() + arr = root.create_dataset( + name="my_dataset", + shape=0, + dtype=object, + object_codec=MsgPack(), + ) + new_items = [ + ["A", 1], + ["B", 2, "hello"], + ] + arr_add = np.empty(len(new_items), dtype=object) + arr_add[:] = new_items + arr.append(arr_add) + + # heterogeneous elements + elem = ["C", 3] + arr[0] = elem + assert arr[0] == elem + + # homogeneous elements + elem = [1, 3] + arr[1] = elem + assert arr[1] == elem From 4bd9764c1857cb563d155ad797011dfd4a44616e Mon Sep 17 00:00:00 2001 From: Joe Hamman Date: Thu, 16 May 2024 17:57:40 -0700 Subject: [PATCH 096/130] release notes for 2.18.1 (#1885) --- docs/release.rst | 15 +++------------ 1 file changed, 3 insertions(+), 12 deletions(-) diff --git a/docs/release.rst b/docs/release.rst index a81be4638f..5ca60b8166 100644 --- a/docs/release.rst +++ b/docs/release.rst @@ -18,25 +18,16 @@ Release notes See `GH1777 `_ for more details on the upcoming 3.0 release. -.. _unreleased: +.. _release_2.18.1: -Unreleased ----------- - -Enhancements -~~~~~~~~~~~~ - -Docs -~~~~ +2.18.1 +------ Maintenance ~~~~~~~~~~~ * Fix a regression when getting or setting a single value from arrays with size-1 chunks. By :user:`Deepak Cherian ` :issue:`1874` -Deprecations -~~~~~~~~~~~~ - .. _release_2.18.0: 2.18.0 From bf895332cdb8f028284f958c58cda773085d5e68 Mon Sep 17 00:00:00 2001 From: Joe Hamman Date: Fri, 17 May 2024 05:52:31 -0700 Subject: [PATCH 097/130] reset release notes (#1886) --- docs/release.rst | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/docs/release.rst b/docs/release.rst index 5ca60b8166..59051bbf97 100644 --- a/docs/release.rst +++ b/docs/release.rst @@ -18,6 +18,23 @@ Release notes See `GH1777 `_ for more details on the upcoming 3.0 release. +.. _unreleased: + +Unreleased +---------- + +Enhancements +~~~~~~~~~~~~ + +Docs +~~~~ + +Maintenance +~~~~~~~~~~~ + +Deprecations +~~~~~~~~~~~~ + .. _release_2.18.1: 2.18.1 From 3e98c3b1d6ed24b7b04fdbaa56ed2c42f5b1ef05 Mon Sep 17 00:00:00 2001 From: Ryan Abernathey Date: Sun, 26 May 2024 14:54:33 -0400 Subject: [PATCH 098/130] Add zstd to old V3 supported codecs (#1914) * add zstd to old V3 supported codecs * get to full test coverage and add release note * fix pre-commit --- docs/release.rst | 3 +++ zarr/codecs.py | 2 +- zarr/meta.py | 4 ++++ zarr/tests/test_meta.py | 23 +++++++++++++++++------ 4 files changed, 25 insertions(+), 7 deletions(-) diff --git a/docs/release.rst b/docs/release.rst index 59051bbf97..48a6c9ca2d 100644 --- a/docs/release.rst +++ b/docs/release.rst @@ -32,6 +32,9 @@ Docs Maintenance ~~~~~~~~~~~ +* Add Zstd codec to old V3 code path. + By :user:`Ryan Abernathey ` + Deprecations ~~~~~~~~~~~~ diff --git a/zarr/codecs.py b/zarr/codecs.py index 4ad68b8627..6fd5e20401 100644 --- a/zarr/codecs.py +++ b/zarr/codecs.py @@ -1,4 +1,4 @@ # flake8: noqa from numcodecs import * -from numcodecs import get_codec, Blosc, Pickle, Zlib, Delta, AsType, BZ2 +from numcodecs import get_codec, Blosc, Pickle, Zlib, Zstd, Delta, AsType, BZ2 from numcodecs.registry import codec_registry diff --git a/zarr/meta.py b/zarr/meta.py index 5430ab305d..44a2b7ebec 100644 --- a/zarr/meta.py +++ b/zarr/meta.py @@ -414,6 +414,8 @@ def _encode_codec_metadata(cls, codec: Codec) -> Optional[Mapping]: uri = uri + "lz4/1.0" elif isinstance(codec, numcodecs.LZMA): uri = uri + "lzma/1.0" + elif isinstance(codec, numcodecs.Zstd): + uri = uri + "zstd/1.0" meta = { "codec": uri, "configuration": config, @@ -439,6 +441,8 @@ def _decode_codec_metadata(cls, meta: Optional[Mapping]) -> Optional[Codec]: conf["id"] = "lz4" elif meta["codec"].startswith(uri + "lzma/"): conf["id"] = "lzma" + elif meta["codec"].startswith(uri + "zstd/"): + conf["id"] = "zstd" else: raise NotImplementedError diff --git a/zarr/tests/test_meta.py b/zarr/tests/test_meta.py index f9010d6788..7b7d526476 100644 --- a/zarr/tests/test_meta.py +++ b/zarr/tests/test_meta.py @@ -5,7 +5,7 @@ import numpy as np import pytest -from zarr.codecs import Blosc, Delta, Pickle, Zlib +from zarr.codecs import Blosc, Delta, Pickle, Zlib, Zstd from zarr.errors import MetadataError from zarr.meta import ( ZARR_FORMAT, @@ -268,17 +268,23 @@ def test_encode_decode_array_dtype_shape(): assert meta_dec["filters"] is None -def test_encode_decode_array_dtype_shape_v3(): +@pytest.mark.parametrize("cname", ["zlib", "zstd"]) +def test_encode_decode_array_dtype_shape_v3(cname): + if cname == "zlib": + compressor = Zlib(1) + elif cname == "zstd": + compressor = Zstd(1) meta = dict( shape=(100,), chunk_grid=dict(type="regular", chunk_shape=(10,), separator=("/")), data_type=np.dtype("(10, 10) Date: Sun, 26 May 2024 12:20:32 -0700 Subject: [PATCH 099/130] doc: update release notes for 2.18.2 (#1915) --- docs/release.rst | 14 +------------- 1 file changed, 1 insertion(+), 13 deletions(-) diff --git a/docs/release.rst b/docs/release.rst index 48a6c9ca2d..6c7ba5139b 100644 --- a/docs/release.rst +++ b/docs/release.rst @@ -18,26 +18,14 @@ Release notes See `GH1777 `_ for more details on the upcoming 3.0 release. -.. _unreleased: - -Unreleased ----------- +.. _release_2.18.2: Enhancements ~~~~~~~~~~~~ -Docs -~~~~ - -Maintenance -~~~~~~~~~~~ - * Add Zstd codec to old V3 code path. By :user:`Ryan Abernathey ` -Deprecations -~~~~~~~~~~~~ - .. _release_2.18.1: 2.18.1 From a054afbbdaf7c8560d0ce5df77af969bc863e335 Mon Sep 17 00:00:00 2001 From: David Stansby Date: Sat, 10 Aug 2024 09:16:40 +0100 Subject: [PATCH 100/130] Drop support for Python 3.9 --- .github/ISSUE_TEMPLATE/bug_report.yml | 2 +- .github/workflows/python-package.yml | 2 +- .github/workflows/releases.yml | 2 +- .readthedocs.yaml | 2 +- docs/release.rst | 10 ++++++++-- pyproject.toml | 4 ++-- 6 files changed, 14 insertions(+), 8 deletions(-) diff --git a/.github/ISSUE_TEMPLATE/bug_report.yml b/.github/ISSUE_TEMPLATE/bug_report.yml index ec98af029e..7b0c4dcfc4 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.yml +++ b/.github/ISSUE_TEMPLATE/bug_report.yml @@ -27,7 +27,7 @@ body: attributes: label: Python Version description: Version of Python interpreter - placeholder: 3.9, 3.10, 3.11, etc. + placeholder: 3.10, 3.11, 3.12 etc. validations: required: true - type: input diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index f53cb2d9a9..61124b8e8e 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -15,7 +15,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: ['3.9', '3.10', '3.11', '3.12'] + python-version: ['3.10', '3.11', '3.12'] numpy_version: ['>=1.24.0', '==1.23.*'] exclude: - python-version: '3.10' diff --git a/.github/workflows/releases.yml b/.github/workflows/releases.yml index 8ac76c899b..f94711ad63 100644 --- a/.github/workflows/releases.yml +++ b/.github/workflows/releases.yml @@ -19,7 +19,7 @@ jobs: - uses: actions/setup-python@v5.1.0 name: Install Python with: - python-version: '3.9' + python-version: '3.11' - name: Install PyBuild run: | diff --git a/.readthedocs.yaml b/.readthedocs.yaml index e45cae1b45..d7190b4771 100644 --- a/.readthedocs.yaml +++ b/.readthedocs.yaml @@ -3,7 +3,7 @@ version: 2 build: os: ubuntu-20.04 tools: - python: "3.9" + python: "3.11" sphinx: configuration: docs/conf.py diff --git a/docs/release.rst b/docs/release.rst index 6c7ba5139b..703fec8df6 100644 --- a/docs/release.rst +++ b/docs/release.rst @@ -24,7 +24,13 @@ Enhancements ~~~~~~~~~~~~ * Add Zstd codec to old V3 code path. - By :user:`Ryan Abernathey ` + By :user:`Ryan Abernathey ` + +Maintenance +~~~~~~~~~~~ + +* Removed support for Python 3.9. + By :user:`David Stansby ` .. _release_2.18.1: @@ -51,7 +57,7 @@ Maintenance * Enable ruff/bugbear rules (B) and fix issues. By :user:`Dimitri Papadopoulos Orfanos ` :issue:`1702`. -* Minor updates to use `np.inf` instead of `np.PINF` / `np.NINF` in preparation for NumPy 2.0.0 release. +* Minor updates to use `np.inf` instead of `np.PINF` / `np.NINF` in preparation for NumPy 2.0.0 release. By :user:`Joe Hamman ` :issue:`1842`. Deprecations diff --git a/pyproject.toml b/pyproject.toml index dacd45ec2c..a0525a8d8a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -10,7 +10,7 @@ readme = { file = "README.md", content-type = "text/markdown" } maintainers = [ { name = "Alistair Miles", email = "alimanfoo@googlemail.com" } ] -requires-python = ">=3.9" +requires-python = ">=3.10" dependencies = [ 'asciitree', 'numpy>=1.23', @@ -30,9 +30,9 @@ classifiers = [ 'Topic :: Software Development :: Libraries :: Python Modules', 'Operating System :: Unix', 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.9', 'Programming Language :: Python :: 3.10', 'Programming Language :: Python :: 3.11', + 'Programming Language :: Python :: 3.12', ] license = { text = "MIT" } From 5437386047323657a7d3ecc5bf775352ef30f800 Mon Sep 17 00:00:00 2001 From: David Stansby Date: Sat, 10 Aug 2024 09:19:06 +0100 Subject: [PATCH 101/130] Revert "Drop support for Python 3.9" This reverts commit a054afbbdaf7c8560d0ce5df77af969bc863e335. --- .github/ISSUE_TEMPLATE/bug_report.yml | 2 +- .github/workflows/python-package.yml | 2 +- .github/workflows/releases.yml | 2 +- .readthedocs.yaml | 2 +- docs/release.rst | 10 ++-------- pyproject.toml | 4 ++-- 6 files changed, 8 insertions(+), 14 deletions(-) diff --git a/.github/ISSUE_TEMPLATE/bug_report.yml b/.github/ISSUE_TEMPLATE/bug_report.yml index 7b0c4dcfc4..ec98af029e 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.yml +++ b/.github/ISSUE_TEMPLATE/bug_report.yml @@ -27,7 +27,7 @@ body: attributes: label: Python Version description: Version of Python interpreter - placeholder: 3.10, 3.11, 3.12 etc. + placeholder: 3.9, 3.10, 3.11, etc. validations: required: true - type: input diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index 61124b8e8e..f53cb2d9a9 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -15,7 +15,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: ['3.10', '3.11', '3.12'] + python-version: ['3.9', '3.10', '3.11', '3.12'] numpy_version: ['>=1.24.0', '==1.23.*'] exclude: - python-version: '3.10' diff --git a/.github/workflows/releases.yml b/.github/workflows/releases.yml index f94711ad63..8ac76c899b 100644 --- a/.github/workflows/releases.yml +++ b/.github/workflows/releases.yml @@ -19,7 +19,7 @@ jobs: - uses: actions/setup-python@v5.1.0 name: Install Python with: - python-version: '3.11' + python-version: '3.9' - name: Install PyBuild run: | diff --git a/.readthedocs.yaml b/.readthedocs.yaml index d7190b4771..e45cae1b45 100644 --- a/.readthedocs.yaml +++ b/.readthedocs.yaml @@ -3,7 +3,7 @@ version: 2 build: os: ubuntu-20.04 tools: - python: "3.11" + python: "3.9" sphinx: configuration: docs/conf.py diff --git a/docs/release.rst b/docs/release.rst index 703fec8df6..6c7ba5139b 100644 --- a/docs/release.rst +++ b/docs/release.rst @@ -24,13 +24,7 @@ Enhancements ~~~~~~~~~~~~ * Add Zstd codec to old V3 code path. - By :user:`Ryan Abernathey ` - -Maintenance -~~~~~~~~~~~ - -* Removed support for Python 3.9. - By :user:`David Stansby ` + By :user:`Ryan Abernathey ` .. _release_2.18.1: @@ -57,7 +51,7 @@ Maintenance * Enable ruff/bugbear rules (B) and fix issues. By :user:`Dimitri Papadopoulos Orfanos ` :issue:`1702`. -* Minor updates to use `np.inf` instead of `np.PINF` / `np.NINF` in preparation for NumPy 2.0.0 release. +* Minor updates to use `np.inf` instead of `np.PINF` / `np.NINF` in preparation for NumPy 2.0.0 release. By :user:`Joe Hamman ` :issue:`1842`. Deprecations diff --git a/pyproject.toml b/pyproject.toml index a0525a8d8a..dacd45ec2c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -10,7 +10,7 @@ readme = { file = "README.md", content-type = "text/markdown" } maintainers = [ { name = "Alistair Miles", email = "alimanfoo@googlemail.com" } ] -requires-python = ">=3.10" +requires-python = ">=3.9" dependencies = [ 'asciitree', 'numpy>=1.23', @@ -30,9 +30,9 @@ classifiers = [ 'Topic :: Software Development :: Libraries :: Python Modules', 'Operating System :: Unix', 'Programming Language :: Python :: 3', + 'Programming Language :: Python :: 3.9', 'Programming Language :: Python :: 3.10', 'Programming Language :: Python :: 3.11', - 'Programming Language :: Python :: 3.12', ] license = { text = "MIT" } From 6888c593a74eedf0d153e763005754e68955bd9f Mon Sep 17 00:00:00 2001 From: Joe Hamman Date: Sun, 11 Aug 2024 04:42:44 -0700 Subject: [PATCH 102/130] Update TEAM.md (#2071) --- TEAM.md | 1 + 1 file changed, 1 insertion(+) diff --git a/TEAM.md b/TEAM.md index 6a22d83d1f..824e57fa7f 100644 --- a/TEAM.md +++ b/TEAM.md @@ -7,6 +7,7 @@ - @jakirkham (jakirkham) - @martindurant (Martin Durant) - @normanrz (Norman Rzepka) +- @dstansby (David Stansby) ## Emeritus core-developers - @alimanfoo (Alistair Miles) From c72d0e2a14caffcde2035e182eb4235ecbdfee1c Mon Sep 17 00:00:00 2001 From: David Stansby Date: Wed, 14 Aug 2024 08:46:38 +0100 Subject: [PATCH 103/130] [v2] Fix doctests with numpy 2.0 (#2073) --- pyproject.toml | 4 ++++ zarr/core.py | 22 +++++++++++----------- zarr/creation.py | 2 +- 3 files changed, 16 insertions(+), 12 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index dacd45ec2c..a5668d0d90 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -145,6 +145,10 @@ filterwarnings = [ "ignore:The .* is deprecated and will be removed in a Zarr-Python version 3*:FutureWarning", "ignore:The experimental Zarr V3 implementation in this version .*:FutureWarning", ] +doctest_subpackage_requires =[ + "zarr/core.py = numpy>=2", + "zarr/creation.py = numpy>=2" +] [tool.codespell] diff --git a/zarr/core.py b/zarr/core.py index b1ccd203db..4f3080e46c 100644 --- a/zarr/core.py +++ b/zarr/core.py @@ -609,11 +609,11 @@ def islice(self, start=None, end=None): Iterate over part of the array: >>> for value in z.islice(25, 30): value; - 25 - 26 - 27 - 28 - 29 + np.int64(25) + np.int64(26) + np.int64(27) + np.int64(28) + np.int64(29) """ if len(self.shape) == 0: @@ -679,7 +679,7 @@ def __getitem__(self, selection): Retrieve a single item:: >>> z[5] - 5 + np.int64(5) Retrieve a region via slicing:: @@ -706,7 +706,7 @@ def __getitem__(self, selection): Retrieve an item:: >>> z[2, 2] - 22 + np.int64(22) Retrieve a region via slicing:: @@ -830,7 +830,7 @@ def get_basic_selection(self, selection=Ellipsis, out=None, fields=None): Retrieve a single item:: >>> z.get_basic_selection(5) - 5 + np.int64(5) Retrieve a region via slicing:: @@ -852,7 +852,7 @@ def get_basic_selection(self, selection=Ellipsis, out=None, fields=None): Retrieve an item:: >>> z.get_basic_selection((2, 2)) - 22 + np.int64(22) Retrieve a region via slicing:: @@ -2819,7 +2819,7 @@ def view( >>> v[:] array([False, False, True, ..., True, False, False]) >>> np.all(a[:].view(dtype=bool) == v[:]) - True + np.True_ An array can be viewed with a dtype with a different item size, however some care is needed to adjust the shape and chunk shape so that chunk @@ -2833,7 +2833,7 @@ def view( >>> v[:10] array([0, 0, 1, 0, 2, 0, 3, 0, 4, 0], dtype=uint8) >>> np.all(a[:].view('u1') == v[:]) - True + np.True_ Change fill value for uninitialized chunks: diff --git a/zarr/creation.py b/zarr/creation.py index 9b2b1d6d4c..f7f3d5a094 100644 --- a/zarr/creation.py +++ b/zarr/creation.py @@ -569,7 +569,7 @@ def open_array( >>> z2 >>> np.all(z1[:] == z2[:]) - True + np.True_ Notes ----- From f231bb2bbbbbc96bfdf3afdb56acb66baeff9699 Mon Sep 17 00:00:00 2001 From: David Stansby Date: Wed, 14 Aug 2024 09:18:51 +0100 Subject: [PATCH 104/130] Fix version number in built docs (#2044) --- docs/conf.py | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/docs/conf.py b/docs/conf.py index 048e77f51d..886160382f 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -16,11 +16,8 @@ import os import sys -# The version info for the project you're documenting, acts as replacement for -# |version| and |release|, also used in various other places throughout the -# built documents. -# -# The short X.Y version. +from importlib.metadata import version as get_version + import zarr # If extensions (or modules to document with autodoc) are in another directory, @@ -75,9 +72,8 @@ copyright = "2024, Zarr Developers" author = "Zarr Developers" -version = zarr.__version__ -# The full version, including alpha/beta/rc tags. -release = zarr.__version__ +version = get_version("zarr") +release = get_version("zarr") # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. From 11fd8db8f06372ffc1f6f7e22315d43e4702ec60 Mon Sep 17 00:00:00 2001 From: Deepak Cherian Date: Wed, 14 Aug 2024 13:48:25 -0600 Subject: [PATCH 105/130] Fix orthogonal indexing with scalar. (#1947) Co-authored-by: David Stansby --- docs/release.rst | 11 +++++++++++ zarr/core.py | 7 ++++++- zarr/tests/test_core.py | 17 +++++++++++++++++ 3 files changed, 34 insertions(+), 1 deletion(-) diff --git a/docs/release.rst b/docs/release.rst index 6c7ba5139b..c40ff5c0d3 100644 --- a/docs/release.rst +++ b/docs/release.rst @@ -18,6 +18,17 @@ Release notes See `GH1777 `_ for more details on the upcoming 3.0 release. +.. _release_2.18.3: + +2.18.3 +------ + +Maintenance +~~~~~~~~~~~ +* Fix a regression when using orthogonal indexing with a scalar. + By :user:`Deepak Cherian ` :issue:`1931` + + .. _release_2.18.2: Enhancements diff --git a/zarr/core.py b/zarr/core.py index 4f3080e46c..141190a0c8 100644 --- a/zarr/core.py +++ b/zarr/core.py @@ -2053,7 +2053,12 @@ def _process_chunk( if isinstance(cdata, UncompressedPartialReadBufferV3): cdata = cdata.read_full() chunk = ensure_ndarray_like(cdata).view(self._dtype) - chunk = chunk.reshape(self._chunks, order=self._order) + # dest.shape is not self._chunks when a dimensions is squeezed out + # For example, assume self._chunks = (5, 5, 1) + # and the selection is [:, :, 0] + # Then out_selection is (slice(5), slice(5)) + # See https://github.com/zarr-developers/zarr-python/issues/1931 + chunk = chunk.reshape(dest.shape, order=self._order) np.copyto(dest, chunk) return diff --git a/zarr/tests/test_core.py b/zarr/tests/test_core.py index 01a78ecd68..4729dc01b6 100644 --- a/zarr/tests/test_core.py +++ b/zarr/tests/test_core.py @@ -3206,3 +3206,20 @@ def test_object_array_indexing(): elem = [1, 3] arr[1] = elem assert arr[1] == elem + + +@pytest.mark.parametrize("shape", ((1, 1, 1), (5, 5, 1), (1, 5, 5))) +def test_scalar_orthogonal_indexing(shape): + # regression test for https://github.com/zarr-developers/zarr-python/issues/1931 + store = zarr.MemoryStore({}) + data = np.random.randint(0, 255, shape) + arr = zarr.zeros( + shape=shape, chunks=shape[:-1] + (1,), compressor=None, store=store, dtype="u1" + ) + arr[:, :, :] = data + store.close() + + zf = zarr.open(store, "r") + assert_array_equal(zf[0, :, :], data[0, :, :]) + assert_array_equal(zf[:, 0, :], data[:, 0, :]) + assert_array_equal(zf[:, :, 0], data[:, :, 0]) From ee43bb861c1910879c5de5a01e24cc2c2516def4 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 16 Aug 2024 09:37:03 +0100 Subject: [PATCH 106/130] Bump the requirements group across 1 directory with 7 updates (#2092) Bumps the requirements group with 7 updates in the / directory: | Package | From | To | | --- | --- | --- | | [numpy](https://github.com/numpy/numpy) | `1.26.4` | `2.0.1` | | [ipywidgets](https://github.com/jupyter-widgets/ipywidgets) | `8.1.2` | `8.1.3` | | [setuptools-scm](https://github.com/pypa/setuptools_scm) | `8.0.4` | `8.1.0` | | [pytest](https://github.com/pytest-dev/pytest) | `8.1.1` | `8.3.2` | | [lmdb](https://github.com/jnwatson/py-lmdb) | `1.4.1` | `1.5.1` | | [redis](https://github.com/redis/redis-py) | `5.0.4` | `5.0.8` | | [pymongo](https://github.com/mongodb/mongo-python-driver) | `4.6.3` | `4.8.0` | Updates `numpy` from 1.26.4 to 2.0.1 - [Release notes](https://github.com/numpy/numpy/releases) - [Changelog](https://github.com/numpy/numpy/blob/main/doc/RELEASE_WALKTHROUGH.rst) - [Commits](https://github.com/numpy/numpy/compare/v1.26.4...v2.0.1) Updates `ipywidgets` from 8.1.2 to 8.1.3 - [Release notes](https://github.com/jupyter-widgets/ipywidgets/releases) - [Commits](https://github.com/jupyter-widgets/ipywidgets/compare/8.1.2...8.1.3) Updates `setuptools-scm` from 8.0.4 to 8.1.0 - [Release notes](https://github.com/pypa/setuptools_scm/releases) - [Changelog](https://github.com/pypa/setuptools_scm/blob/main/CHANGELOG.md) - [Commits](https://github.com/pypa/setuptools_scm/compare/v8.0.4...v8.1.0) Updates `pytest` from 8.1.1 to 8.3.2 - [Release notes](https://github.com/pytest-dev/pytest/releases) - [Changelog](https://github.com/pytest-dev/pytest/blob/main/CHANGELOG.rst) - [Commits](https://github.com/pytest-dev/pytest/compare/8.1.1...8.3.2) Updates `lmdb` from 1.4.1 to 1.5.1 - [Changelog](https://github.com/jnwatson/py-lmdb/blob/master/ChangeLog) - [Commits](https://github.com/jnwatson/py-lmdb/compare/py-lmdb_1.4.1...py-lmdb_1.5.1) Updates `redis` from 5.0.4 to 5.0.8 - [Release notes](https://github.com/redis/redis-py/releases) - [Changelog](https://github.com/redis/redis-py/blob/master/CHANGES) - [Commits](https://github.com/redis/redis-py/compare/v5.0.4...v5.0.8) Updates `pymongo` from 4.6.3 to 4.8.0 - [Release notes](https://github.com/mongodb/mongo-python-driver/releases) - [Changelog](https://github.com/mongodb/mongo-python-driver/blob/master/doc/changelog.rst) - [Commits](https://github.com/mongodb/mongo-python-driver/compare/4.6.3...4.8.0) --- updated-dependencies: - dependency-name: numpy dependency-type: direct:development update-type: version-update:semver-major dependency-group: requirements - dependency-name: ipywidgets dependency-type: direct:development update-type: version-update:semver-patch dependency-group: requirements - dependency-name: setuptools-scm dependency-type: direct:development update-type: version-update:semver-minor dependency-group: requirements - dependency-name: pytest dependency-type: direct:development update-type: version-update:semver-minor dependency-group: requirements - dependency-name: lmdb dependency-type: direct:development update-type: version-update:semver-minor dependency-group: requirements - dependency-name: redis dependency-type: direct:development update-type: version-update:semver-patch dependency-group: requirements - dependency-name: pymongo dependency-type: direct:development update-type: version-update:semver-minor dependency-group: requirements ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- requirements_dev_minimal.txt | 4 ++-- requirements_dev_numpy.txt | 2 +- requirements_dev_optional.txt | 8 ++++---- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/requirements_dev_minimal.txt b/requirements_dev_minimal.txt index 5d156db655..df96c608b6 100644 --- a/requirements_dev_minimal.txt +++ b/requirements_dev_minimal.txt @@ -3,6 +3,6 @@ asciitree==0.3.3 fasteners==0.19 numcodecs==0.12.1 msgpack-python==0.5.6 -setuptools-scm==8.0.4 +setuptools-scm==8.1.0 # test requirements -pytest==8.1.1 +pytest==8.3.2 diff --git a/requirements_dev_numpy.txt b/requirements_dev_numpy.txt index d8d6c3d097..d53ec456a7 100644 --- a/requirements_dev_numpy.txt +++ b/requirements_dev_numpy.txt @@ -1,4 +1,4 @@ # Break this out into a separate file to allow testing against # different versions of numpy. This file should pin to the latest # numpy version. -numpy==1.26.4 +numpy==2.0.1 diff --git a/requirements_dev_optional.txt b/requirements_dev_optional.txt index 3456cca21a..87c9538a3c 100644 --- a/requirements_dev_optional.txt +++ b/requirements_dev_optional.txt @@ -1,17 +1,17 @@ # optional library requirements # bsddb3==6.2.6; sys_platform != 'win32' -lmdb==1.4.1; sys_platform != 'win32' +lmdb==1.5.1; sys_platform != 'win32' # optional library requirements for Jupyter ipytree==0.2.2 -ipywidgets==8.1.2 +ipywidgets==8.1.3 # optional library requirements for services # don't let pyup change pinning for azure-storage-blob, need to pin to older # version to get compatibility with azure storage emulator on appveyor (FIXME) azure-storage-blob==12.16.0 # pyup: ignore -redis==5.0.4 +redis==5.0.8 types-redis types-setuptools -pymongo==4.6.3 +pymongo==4.8.0 # optional test requirements coverage pytest-cov==5.0.0 From addc39dfc53fd0f992f59196f91e78b81395f494 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 16 Aug 2024 09:37:27 +0100 Subject: [PATCH 107/130] Bump the actions group with 2 updates (#2087) Bumps the actions group with 2 updates: [actions/setup-python](https://github.com/actions/setup-python) and [pypa/gh-action-pypi-publish](https://github.com/pypa/gh-action-pypi-publish). Updates `actions/setup-python` from 5.1.0 to 5.1.1 - [Release notes](https://github.com/actions/setup-python/releases) - [Commits](https://github.com/actions/setup-python/compare/v5.1.0...v5.1.1) Updates `pypa/gh-action-pypi-publish` from 1.8.14 to 1.9.0 - [Release notes](https://github.com/pypa/gh-action-pypi-publish/releases) - [Commits](https://github.com/pypa/gh-action-pypi-publish/compare/v1.8.14...v1.9.0) --- updated-dependencies: - dependency-name: actions/setup-python dependency-type: direct:production update-type: version-update:semver-patch dependency-group: actions - dependency-name: pypa/gh-action-pypi-publish dependency-type: direct:production update-type: version-update:semver-minor dependency-group: actions ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/releases.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/releases.yml b/.github/workflows/releases.yml index 8ac76c899b..20cb5890b4 100644 --- a/.github/workflows/releases.yml +++ b/.github/workflows/releases.yml @@ -16,7 +16,7 @@ jobs: submodules: true fetch-depth: 0 - - uses: actions/setup-python@v5.1.0 + - uses: actions/setup-python@v5.1.1 name: Install Python with: python-version: '3.9' @@ -64,7 +64,7 @@ jobs: with: name: releases path: dist - - uses: pypa/gh-action-pypi-publish@v1.8.14 + - uses: pypa/gh-action-pypi-publish@v1.9.0 with: user: __token__ password: ${{ secrets.pypi_password }} From c5c4698baed2f74a545c163950ce8968306cfc99 Mon Sep 17 00:00:00 2001 From: David Stansby Date: Sat, 17 Aug 2024 08:50:21 +0100 Subject: [PATCH 108/130] [v2] Drop support for Python 3.9 (#2074) * Drop support for Python 3.9 * Ignore B905 ruff rule * Fix release notes --- .github/ISSUE_TEMPLATE/bug_report.yml | 2 +- .github/workflows/python-package.yml | 2 +- .github/workflows/releases.yml | 2 +- .readthedocs.yaml | 2 +- docs/release.rst | 11 +++++++++-- pyproject.toml | 5 +++-- 6 files changed, 16 insertions(+), 8 deletions(-) diff --git a/.github/ISSUE_TEMPLATE/bug_report.yml b/.github/ISSUE_TEMPLATE/bug_report.yml index ec98af029e..7b0c4dcfc4 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.yml +++ b/.github/ISSUE_TEMPLATE/bug_report.yml @@ -27,7 +27,7 @@ body: attributes: label: Python Version description: Version of Python interpreter - placeholder: 3.9, 3.10, 3.11, etc. + placeholder: 3.10, 3.11, 3.12 etc. validations: required: true - type: input diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index f53cb2d9a9..61124b8e8e 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -15,7 +15,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: ['3.9', '3.10', '3.11', '3.12'] + python-version: ['3.10', '3.11', '3.12'] numpy_version: ['>=1.24.0', '==1.23.*'] exclude: - python-version: '3.10' diff --git a/.github/workflows/releases.yml b/.github/workflows/releases.yml index 20cb5890b4..ec0601f0f4 100644 --- a/.github/workflows/releases.yml +++ b/.github/workflows/releases.yml @@ -19,7 +19,7 @@ jobs: - uses: actions/setup-python@v5.1.1 name: Install Python with: - python-version: '3.9' + python-version: '3.11' - name: Install PyBuild run: | diff --git a/.readthedocs.yaml b/.readthedocs.yaml index e45cae1b45..d7190b4771 100644 --- a/.readthedocs.yaml +++ b/.readthedocs.yaml @@ -3,7 +3,7 @@ version: 2 build: os: ubuntu-20.04 tools: - python: "3.9" + python: "3.11" sphinx: configuration: docs/conf.py diff --git a/docs/release.rst b/docs/release.rst index c40ff5c0d3..9c2840a70a 100644 --- a/docs/release.rst +++ b/docs/release.rst @@ -18,6 +18,7 @@ Release notes See `GH1777 `_ for more details on the upcoming 3.0 release. + .. _release_2.18.3: 2.18.3 @@ -25,17 +26,23 @@ Release notes Maintenance ~~~~~~~~~~~ +* Removed support for Python 3.9. + By :user:`David Stansby ` + * Fix a regression when using orthogonal indexing with a scalar. By :user:`Deepak Cherian ` :issue:`1931` .. _release_2.18.2: +2.18.2 +------ + Enhancements ~~~~~~~~~~~~ * Add Zstd codec to old V3 code path. - By :user:`Ryan Abernathey ` + By :user:`Ryan Abernathey ` .. _release_2.18.1: @@ -62,7 +69,7 @@ Maintenance * Enable ruff/bugbear rules (B) and fix issues. By :user:`Dimitri Papadopoulos Orfanos ` :issue:`1702`. -* Minor updates to use `np.inf` instead of `np.PINF` / `np.NINF` in preparation for NumPy 2.0.0 release. +* Minor updates to use `np.inf` instead of `np.PINF` / `np.NINF` in preparation for NumPy 2.0.0 release. By :user:`Joe Hamman ` :issue:`1842`. Deprecations diff --git a/pyproject.toml b/pyproject.toml index a5668d0d90..da5fb34d89 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -10,7 +10,7 @@ readme = { file = "README.md", content-type = "text/markdown" } maintainers = [ { name = "Alistair Miles", email = "alimanfoo@googlemail.com" } ] -requires-python = ">=3.9" +requires-python = ">=3.10" dependencies = [ 'asciitree', 'numpy>=1.23', @@ -30,9 +30,9 @@ classifiers = [ 'Topic :: Software Development :: Libraries :: Python Modules', 'Operating System :: Unix', 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.9', 'Programming Language :: Python :: 3.10', 'Programming Language :: Python :: 3.11', + 'Programming Language :: Python :: 3.12', ] license = { text = "MIT" } @@ -107,6 +107,7 @@ exclude = [ extend-select = [ "B" ] +ignore = ["B905"] # zip-without-explicit-strict [tool.black] line-length = 100 From 35e8106278e94edd91dad370bab33f5633167044 Mon Sep 17 00:00:00 2001 From: David Stansby Date: Fri, 23 Aug 2024 12:02:29 +0100 Subject: [PATCH 109/130] Fix Array.__array__ for numpy 2.1 (#2106) * Fix Array.__array__ for numpy 2.1 * Add changelog * Depend on np.array for array coercions --- docs/release.rst | 11 ++++++++++- zarr/core.py | 7 ++----- 2 files changed, 12 insertions(+), 6 deletions(-) diff --git a/docs/release.rst b/docs/release.rst index 9c2840a70a..697dfde3b8 100644 --- a/docs/release.rst +++ b/docs/release.rst @@ -24,14 +24,23 @@ Release notes 2.18.3 ------ +Enhancements +~~~~~~~~~~~~ +* Added support for creating a copy of data when converting a `zarr.Array` + to a numpy array. + By :user:`David Stansby ` + Maintenance ~~~~~~~~~~~ * Removed support for Python 3.9. By :user:`David Stansby ` - + * Fix a regression when using orthogonal indexing with a scalar. By :user:`Deepak Cherian ` :issue:`1931` +* Added compatibility with numpy 2.1. + By :user:`David Stansby ` + .. _release_2.18.2: diff --git a/zarr/core.py b/zarr/core.py index 141190a0c8..08234e193c 100644 --- a/zarr/core.py +++ b/zarr/core.py @@ -575,11 +575,8 @@ def __eq__(self, other): # store comparison ) - def __array__(self, *args): - a = self[...] - if args: - a = a.astype(args[0]) - return a + def __array__(self, dtype=None, copy=None): + return np.array(self[...], dtype=dtype, copy=copy) def islice(self, start=None, end=None): """ From 025e1031fc43b6e672b0dc9a53953afb1b9eef5d Mon Sep 17 00:00:00 2001 From: David Stansby Date: Mon, 26 Aug 2024 16:57:00 +0100 Subject: [PATCH 110/130] Bump test version of numcodecs (#2114) * Bump test version of numcodecs * Fix test for numcodecs 0.13 --- requirements_dev_minimal.txt | 2 +- zarr/tests/test_meta.py | 34 +++++++++++++--------------------- 2 files changed, 14 insertions(+), 22 deletions(-) diff --git a/requirements_dev_minimal.txt b/requirements_dev_minimal.txt index df96c608b6..2991d99299 100644 --- a/requirements_dev_minimal.txt +++ b/requirements_dev_minimal.txt @@ -1,7 +1,7 @@ # library requirements asciitree==0.3.3 fasteners==0.19 -numcodecs==0.12.1 +numcodecs==0.13.0 msgpack-python==0.5.6 setuptools-scm==8.1.0 # test requirements diff --git a/zarr/tests/test_meta.py b/zarr/tests/test_meta.py index 7b7d526476..57ab9a0781 100644 --- a/zarr/tests/test_meta.py +++ b/zarr/tests/test_meta.py @@ -282,36 +282,28 @@ def test_encode_decode_array_dtype_shape_v3(cname): fill_value=None, chunk_memory_layout="C", ) - - meta_json = ( - """{ + meta_expected = { "attributes": {}, - "chunk_grid": { - "chunk_shape": [10], - "separator": "/", - "type": "regular" - }, + "chunk_grid": {"chunk_shape": [10], "separator": "/", "type": "regular"}, "chunk_memory_layout": "C", "compressor": { - """ - + f""" - "codec": "https://purl.org/zarr/spec/codec/{cname}/1.0", - """ - + """ - "configuration": { - "level": 1 - } + "codec": f"https://purl.org/zarr/spec/codec/{cname}/1.0", + "configuration": {"level": 1}, }, "data_type": " Date: Mon, 26 Aug 2024 12:56:20 -0700 Subject: [PATCH 111/130] fix: numpy 1.24 compat for Array.__array__ (#2123) --- zarr/core.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/zarr/core.py b/zarr/core.py index 08234e193c..d13da27bc6 100644 --- a/zarr/core.py +++ b/zarr/core.py @@ -575,8 +575,8 @@ def __eq__(self, other): # store comparison ) - def __array__(self, dtype=None, copy=None): - return np.array(self[...], dtype=dtype, copy=copy) + def __array__(self, *args, **kwargs): + return np.array(self[...], *args, **kwargs) def islice(self, start=None, end=None): """ From 2e928b145c5ffdb088c9973641d71ac2937e5e49 Mon Sep 17 00:00:00 2001 From: Joe Hamman Date: Mon, 26 Aug 2024 14:32:22 -0700 Subject: [PATCH 112/130] Deprecate N5Store (#2103) * deprecate(n5): add deprecation warning to N5Store * also deprecate N5FSStore * docs * fix doc --- docs/release.rst | 11 +++++++++-- zarr/n5.py | 22 ++++++++++++++++++++++ 2 files changed, 31 insertions(+), 2 deletions(-) diff --git a/docs/release.rst b/docs/release.rst index 697dfde3b8..05110f8ce1 100644 --- a/docs/release.rst +++ b/docs/release.rst @@ -28,12 +28,13 @@ Enhancements ~~~~~~~~~~~~ * Added support for creating a copy of data when converting a `zarr.Array` to a numpy array. - By :user:`David Stansby ` + By :user:`David Stansby ` (:issue:`2106`) and + :user:`Joe Hamman ` (:issue:`2123`). Maintenance ~~~~~~~~~~~ * Removed support for Python 3.9. - By :user:`David Stansby ` + By :user:`David Stansby ` (:issue:`2074`). * Fix a regression when using orthogonal indexing with a scalar. By :user:`Deepak Cherian ` :issue:`1931` @@ -41,6 +42,12 @@ Maintenance * Added compatibility with numpy 2.1. By :user:`David Stansby ` +Deprecations +~~~~~~~~~~~~ + +* Deprecate :class:`zarr.n5.N5Store` and :class:`zarr.n5.N5FSStore`. These + stores are slated to be removed in Zarr Python 3.0. + By :user:`Joe Hamman ` :issue:`2085`. .. _release_2.18.2: diff --git a/zarr/n5.py b/zarr/n5.py index 3d3e9afa26..3bb7093128 100644 --- a/zarr/n5.py +++ b/zarr/n5.py @@ -69,8 +69,20 @@ class N5Store(NestedDirectoryStore): Safe to write in multiple threads or processes. + .. deprecated:: 2.18.3 + `N5Store` will be removed in Zarr 3.0.0. """ + def __init__(self, *args: Any, **kwargs: Any) -> None: + super().__init__(*args, **kwargs) + warnings.warn( + "The N5Store is deprecated and will be removed in a Zarr-Python version 3, " + "see https://github.com/zarr-developers/zarr-python/issues/1274 and " + "https://github.com/zarr-developers/n5py for more information.", + FutureWarning, + stacklevel=2, + ) + def __getitem__(self, key: str) -> bytes: if key.endswith(zarr_group_meta_key): key_new = key.replace(zarr_group_meta_key, n5_attrs_key) @@ -322,6 +334,9 @@ class N5FSStore(FSStore): storage, and this procedure requires chunk keys with "." separated dimensions, hence the Zarr arrays targeting N5 have the deceptive "." dimension separator. + + .. deprecated:: 2.18.3 + `N5FSStore` will be removed in Zarr 3.0.0. """ _array_meta_key = "attributes.json" @@ -329,6 +344,13 @@ class N5FSStore(FSStore): _attrs_key = "attributes.json" def __init__(self, *args, **kwargs): + warnings.warn( + "The N5FSStore is deprecated and will be removed in a Zarr-Python version 3, " + "see https://github.com/zarr-developers/zarr-python/issues/1274 and " + "https://github.com/zarr-developers/n5py for more information.", + FutureWarning, + stacklevel=2, + ) if "dimension_separator" in kwargs: kwargs.pop("dimension_separator") warnings.warn( From 5bf57bd7492159823f83937fc7ccab2fdb61eeb5 Mon Sep 17 00:00:00 2001 From: David Stansby Date: Mon, 26 Aug 2024 23:31:38 +0100 Subject: [PATCH 113/130] Run tests on numpy 1.23 (#2124) --- .github/workflows/python-package.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index 61124b8e8e..f50fa7d9e0 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -18,8 +18,6 @@ jobs: python-version: ['3.10', '3.11', '3.12'] numpy_version: ['>=1.24.0', '==1.23.*'] exclude: - - python-version: '3.10' - numpy_version: '==1.23.*' - python-version: '3.11' numpy_version: '==1.23.*' - python-version: '3.12' From 9e0b5e88d9473fb3403a9cb35eff0526a75197cb Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 27 Aug 2024 17:37:09 +0100 Subject: [PATCH 114/130] Bump the requirements group across 1 directory with 3 updates (#2129) Bumps the requirements group with 3 updates in the / directory: [numpy](https://github.com/numpy/numpy), [ipywidgets](https://github.com/jupyter-widgets/ipywidgets) and [azure-storage-blob](https://github.com/Azure/azure-sdk-for-python). Updates `numpy` from 2.0.1 to 2.1.0 - [Release notes](https://github.com/numpy/numpy/releases) - [Changelog](https://github.com/numpy/numpy/blob/main/doc/RELEASE_WALKTHROUGH.rst) - [Commits](https://github.com/numpy/numpy/compare/v2.0.1...v2.1.0) Updates `ipywidgets` from 8.1.3 to 8.1.5 - [Release notes](https://github.com/jupyter-widgets/ipywidgets/releases) - [Commits](https://github.com/jupyter-widgets/ipywidgets/compare/8.1.3...8.1.5) Updates `azure-storage-blob` from 12.16.0 to 12.21.0 - [Release notes](https://github.com/Azure/azure-sdk-for-python/releases) - [Changelog](https://github.com/Azure/azure-sdk-for-python/blob/main/doc/esrp_release.md) - [Commits](https://github.com/Azure/azure-sdk-for-python/compare/azure-storage-blob_12.16.0...azure-storage-blob_12.21.0) --- updated-dependencies: - dependency-name: numpy dependency-type: direct:development update-type: version-update:semver-minor dependency-group: requirements - dependency-name: ipywidgets dependency-type: direct:development update-type: version-update:semver-patch dependency-group: requirements - dependency-name: azure-storage-blob dependency-type: direct:development update-type: version-update:semver-minor dependency-group: requirements ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- requirements_dev_numpy.txt | 2 +- requirements_dev_optional.txt | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/requirements_dev_numpy.txt b/requirements_dev_numpy.txt index d53ec456a7..6423997a29 100644 --- a/requirements_dev_numpy.txt +++ b/requirements_dev_numpy.txt @@ -1,4 +1,4 @@ # Break this out into a separate file to allow testing against # different versions of numpy. This file should pin to the latest # numpy version. -numpy==2.0.1 +numpy==2.1.0 diff --git a/requirements_dev_optional.txt b/requirements_dev_optional.txt index 87c9538a3c..9d75d78e44 100644 --- a/requirements_dev_optional.txt +++ b/requirements_dev_optional.txt @@ -3,11 +3,11 @@ lmdb==1.5.1; sys_platform != 'win32' # optional library requirements for Jupyter ipytree==0.2.2 -ipywidgets==8.1.3 +ipywidgets==8.1.5 # optional library requirements for services # don't let pyup change pinning for azure-storage-blob, need to pin to older # version to get compatibility with azure storage emulator on appveyor (FIXME) -azure-storage-blob==12.16.0 # pyup: ignore +azure-storage-blob==12.21.0 # pyup: ignore redis==5.0.8 types-redis types-setuptools From cc2cdeeccca4a0db2adf4bcde0b7f91b2100bbec Mon Sep 17 00:00:00 2001 From: Joe Hamman Date: Tue, 27 Aug 2024 10:10:37 -0700 Subject: [PATCH 115/130] chore: bump minimum numpy version to 1.24 (#2127) * chore: bump minimum version to 1.24 * Update .github/workflows/python-package.yml --- .github/workflows/python-package.yml | 8 +++----- docs/release.rst | 5 ++++- pyproject.toml | 2 +- 3 files changed, 8 insertions(+), 7 deletions(-) diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index f50fa7d9e0..76ae9f982e 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -16,12 +16,10 @@ jobs: strategy: matrix: python-version: ['3.10', '3.11', '3.12'] - numpy_version: ['>=1.24.0', '==1.23.*'] + numpy_version: ['>=2.1', '==1.24.*'] exclude: - - python-version: '3.11' - numpy_version: '==1.23.*' - python-version: '3.12' - numpy_version: '==1.23.*' + numpy_version: '==1.24.*' services: redis: image: redis @@ -61,7 +59,7 @@ jobs: conda activate zarr-env python -m pip install --upgrade pip python -m pip install -U pip setuptools wheel line_profiler - python -m pip install -rrequirements_dev_minimal.txt numpy${{matrix.numpy_version}} -rrequirements_dev_optional.txt pymongo redis + python -m pip install -r requirements_dev_minimal.txt numpy${{matrix.numpy_version}} -r requirements_dev_optional.txt pymongo redis python -m pip install -e . python -m pip freeze - name: Tests diff --git a/docs/release.rst b/docs/release.rst index 05110f8ce1..69616d6052 100644 --- a/docs/release.rst +++ b/docs/release.rst @@ -39,9 +39,12 @@ Maintenance * Fix a regression when using orthogonal indexing with a scalar. By :user:`Deepak Cherian ` :issue:`1931` -* Added compatibility with numpy 2.1. +* Added compatibility with NumPy 2.1. By :user:`David Stansby ` +* Bump minimum NumPy version to 1.24. + :user:`Joe Hamman ` (:issue:`2127`). + Deprecations ~~~~~~~~~~~~ diff --git a/pyproject.toml b/pyproject.toml index da5fb34d89..ec06b63a96 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -13,7 +13,7 @@ maintainers = [ requires-python = ">=3.10" dependencies = [ 'asciitree', - 'numpy>=1.23', + 'numpy>=1.24', 'fasteners; sys_platform != "emscripten"', 'numcodecs>=0.10.0', ] From cb2c669526c4ba78a8b2f077de9f3cdbc356e56a Mon Sep 17 00:00:00 2001 From: David Stansby Date: Thu, 29 Aug 2024 13:12:17 +0100 Subject: [PATCH 116/130] Remove un-needed package installs in CI (#2095) --- .github/workflows/python-package.yml | 5 ++--- .github/workflows/windows-testing.yml | 3 +-- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index 76ae9f982e..b1be7e425d 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -50,7 +50,7 @@ jobs: - name: Create Conda environment with the rights deps shell: "bash -l {0}" run: | - conda create -n zarr-env python==${{matrix.python-version}} bsddb3 numcodecs lmdb pip nodejs flake8 mypy + conda create -n zarr-env python==${{matrix.python-version}} bsddb3 pip nodejs conda activate zarr-env npm install -g azurite - name: Install dependencies @@ -58,8 +58,7 @@ jobs: run: | conda activate zarr-env python -m pip install --upgrade pip - python -m pip install -U pip setuptools wheel line_profiler - python -m pip install -r requirements_dev_minimal.txt numpy${{matrix.numpy_version}} -r requirements_dev_optional.txt pymongo redis + python -m pip install -r requirements_dev_minimal.txt numpy${{matrix.numpy_version}} -r requirements_dev_optional.txt line_profiler pymongo redis python -m pip install -e . python -m pip freeze - name: Tests diff --git a/.github/workflows/windows-testing.yml b/.github/workflows/windows-testing.yml index ab86831aae..1e22fec6d1 100644 --- a/.github/workflows/windows-testing.yml +++ b/.github/workflows/windows-testing.yml @@ -31,13 +31,12 @@ jobs: - name: Create Conda environment with the rights deps shell: bash -l {0} run: | - conda create -n zarr-env python==${{matrix.python-version}} numcodecs pip nodejs + conda create -n zarr-env python==${{matrix.python-version}} pip nodejs - name: Install dependencies shell: bash -l {0} run: | conda activate zarr-env python -m pip install --upgrade pip - python -m pip install -U pip setuptools wheel python -m pip install -r requirements_dev_numpy.txt -r requirements_dev_minimal.txt -r requirements_dev_optional.txt python -m pip install . python -m pip freeze From 6656d1a2d8f95dfb29724327d9dcff69667dc9d7 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 2 Sep 2024 11:41:00 +0100 Subject: [PATCH 117/130] Bump the actions group with 2 updates (#2146) Bumps the actions group with 2 updates: [actions/setup-python](https://github.com/actions/setup-python) and [pypa/gh-action-pypi-publish](https://github.com/pypa/gh-action-pypi-publish). Updates `actions/setup-python` from 5.1.1 to 5.2.0 - [Release notes](https://github.com/actions/setup-python/releases) - [Commits](https://github.com/actions/setup-python/compare/v5.1.1...v5.2.0) Updates `pypa/gh-action-pypi-publish` from 1.9.0 to 1.10.0 - [Release notes](https://github.com/pypa/gh-action-pypi-publish/releases) - [Commits](https://github.com/pypa/gh-action-pypi-publish/compare/v1.9.0...v1.10.0) --- updated-dependencies: - dependency-name: actions/setup-python dependency-type: direct:production update-type: version-update:semver-minor dependency-group: actions - dependency-name: pypa/gh-action-pypi-publish dependency-type: direct:production update-type: version-update:semver-minor dependency-group: actions ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/releases.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/releases.yml b/.github/workflows/releases.yml index ec0601f0f4..9f115ba3e6 100644 --- a/.github/workflows/releases.yml +++ b/.github/workflows/releases.yml @@ -16,7 +16,7 @@ jobs: submodules: true fetch-depth: 0 - - uses: actions/setup-python@v5.1.1 + - uses: actions/setup-python@v5.2.0 name: Install Python with: python-version: '3.11' @@ -64,7 +64,7 @@ jobs: with: name: releases path: dist - - uses: pypa/gh-action-pypi-publish@v1.9.0 + - uses: pypa/gh-action-pypi-publish@v1.10.0 with: user: __token__ password: ${{ secrets.pypi_password }} From f1978dde52b11a7803a1a06937e69948acc8fe84 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 4 Sep 2024 12:23:21 +0100 Subject: [PATCH 118/130] Bump numpy from 2.1.0 to 2.1.1 in the requirements group (#2151) Bumps the requirements group with 1 update: [numpy](https://github.com/numpy/numpy). Updates `numpy` from 2.1.0 to 2.1.1 - [Release notes](https://github.com/numpy/numpy/releases) - [Changelog](https://github.com/numpy/numpy/blob/main/doc/RELEASE_WALKTHROUGH.rst) - [Commits](https://github.com/numpy/numpy/compare/v2.1.0...v2.1.1) --- updated-dependencies: - dependency-name: numpy dependency-type: direct:development update-type: version-update:semver-patch dependency-group: requirements ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- requirements_dev_numpy.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements_dev_numpy.txt b/requirements_dev_numpy.txt index 6423997a29..190610d3d3 100644 --- a/requirements_dev_numpy.txt +++ b/requirements_dev_numpy.txt @@ -1,4 +1,4 @@ # Break this out into a separate file to allow testing against # different versions of numpy. This file should pin to the latest # numpy version. -numpy==2.1.0 +numpy==2.1.1 From 88b2100ed3f0fed4c5b0a0bc38d29348d44e1ea0 Mon Sep 17 00:00:00 2001 From: Joe Hamman Date: Thu, 5 Sep 2024 05:40:20 -0700 Subject: [PATCH 119/130] chore(docs): update release notes ahead of 2.18.4 (#2152) --- docs/release.rst | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/docs/release.rst b/docs/release.rst index 69616d6052..a62d6a653c 100644 --- a/docs/release.rst +++ b/docs/release.rst @@ -18,6 +18,19 @@ Release notes See `GH1777 `_ for more details on the upcoming 3.0 release. +.. _release_2.18.4: + +2.18.4 (unreleased) +------------------- + +Enhancements +~~~~~~~~~~~~ + +Maintenance +~~~~~~~~~~~ + +Deprecations +~~~~~~~~~~~~ .. _release_2.18.3: From aa9da2b63e6d7ba35774f232b1e66be0575a8a3f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 9 Sep 2024 16:34:50 +0100 Subject: [PATCH 120/130] Bump pypa/gh-action-pypi-publish from 1.10.0 to 1.10.1 in the actions group (#2161) Bumps the actions group with 1 update: [pypa/gh-action-pypi-publish](https://github.com/pypa/gh-action-pypi-publish). Updates `pypa/gh-action-pypi-publish` from 1.10.0 to 1.10.1 - [Release notes](https://github.com/pypa/gh-action-pypi-publish/releases) - [Commits](https://github.com/pypa/gh-action-pypi-publish/compare/v1.10.0...v1.10.1) --- updated-dependencies: - dependency-name: pypa/gh-action-pypi-publish dependency-type: direct:production update-type: version-update:semver-patch dependency-group: actions ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/releases.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/releases.yml b/.github/workflows/releases.yml index 9f115ba3e6..2d7cbd49b6 100644 --- a/.github/workflows/releases.yml +++ b/.github/workflows/releases.yml @@ -64,7 +64,7 @@ jobs: with: name: releases path: dist - - uses: pypa/gh-action-pypi-publish@v1.10.0 + - uses: pypa/gh-action-pypi-publish@v1.10.1 with: user: __token__ password: ${{ secrets.pypi_password }} From 774f5b1e5a84e7cea6b6dfb913c4330854e712b8 Mon Sep 17 00:00:00 2001 From: Jonny Saunders Date: Tue, 10 Sep 2024 07:09:08 -0700 Subject: [PATCH 121/130] [docs] remove primary sidebar from tutorial (#2142) * remove primary sidebar from tutorial * lint --- docs/conf.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/conf.py b/docs/conf.py index 886160382f..136fcf32d6 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -195,7 +195,7 @@ def setup(app): # html_use_smartypants = True # Custom sidebar templates, maps document names to template names. -# html_sidebars = {} +html_sidebars = {"tutorial": []} # Additional templates that should be rendered to pages, maps page names to # template names. From e1d98cdf3d35e4cb2815912d82a64c1623347c87 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 10 Sep 2024 23:35:06 -0700 Subject: [PATCH 122/130] Bump pytest from 8.3.2 to 8.3.3 in the requirements group (#2172) Bumps the requirements group with 1 update: [pytest](https://github.com/pytest-dev/pytest). Updates `pytest` from 8.3.2 to 8.3.3 - [Release notes](https://github.com/pytest-dev/pytest/releases) - [Changelog](https://github.com/pytest-dev/pytest/blob/main/CHANGELOG.rst) - [Commits](https://github.com/pytest-dev/pytest/compare/8.3.2...8.3.3) --- updated-dependencies: - dependency-name: pytest dependency-type: direct:development update-type: version-update:semver-patch dependency-group: requirements ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- requirements_dev_minimal.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements_dev_minimal.txt b/requirements_dev_minimal.txt index 2991d99299..1d29404d6c 100644 --- a/requirements_dev_minimal.txt +++ b/requirements_dev_minimal.txt @@ -5,4 +5,4 @@ numcodecs==0.13.0 msgpack-python==0.5.6 setuptools-scm==8.1.0 # test requirements -pytest==8.3.2 +pytest==8.3.3 From a0110796ac246d2996e9e8bffcbf5a436b0d9bf6 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 23 Sep 2024 21:47:16 -0600 Subject: [PATCH 123/130] Bump pypa/gh-action-pypi-publish in the actions group (#2220) Bumps the actions group with 1 update: [pypa/gh-action-pypi-publish](https://github.com/pypa/gh-action-pypi-publish). Updates `pypa/gh-action-pypi-publish` from 1.10.1 to 1.10.2 - [Release notes](https://github.com/pypa/gh-action-pypi-publish/releases) - [Commits](https://github.com/pypa/gh-action-pypi-publish/compare/v1.10.1...v1.10.2) --- updated-dependencies: - dependency-name: pypa/gh-action-pypi-publish dependency-type: direct:production update-type: version-update:semver-patch dependency-group: actions ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/releases.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/releases.yml b/.github/workflows/releases.yml index 2d7cbd49b6..dccfbc208f 100644 --- a/.github/workflows/releases.yml +++ b/.github/workflows/releases.yml @@ -64,7 +64,7 @@ jobs: with: name: releases path: dist - - uses: pypa/gh-action-pypi-publish@v1.10.1 + - uses: pypa/gh-action-pypi-publish@v1.10.2 with: user: __token__ password: ${{ secrets.pypi_password }} From 53f600b2e8fed934202f60d3ed4f1a23395ea248 Mon Sep 17 00:00:00 2001 From: Joe Hamman Date: Tue, 24 Sep 2024 10:34:33 -0700 Subject: [PATCH 124/130] Update TEAM.md (#2227) --- TEAM.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/TEAM.md b/TEAM.md index 824e57fa7f..e6975d7c04 100644 --- a/TEAM.md +++ b/TEAM.md @@ -8,6 +8,8 @@ - @martindurant (Martin Durant) - @normanrz (Norman Rzepka) - @dstansby (David Stansby) +- @dcherian (Deepak Cherian) +- @TomAugspurger (Tom Augspurger) ## Emeritus core-developers - @alimanfoo (Alistair Miles) From a9abbe7b3e6dcd6540a89a18dec1ed67a273fac6 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 1 Oct 2024 18:28:21 +0200 Subject: [PATCH 125/130] Bump the requirements group across 1 directory with 3 updates (#2284) Bumps the requirements group with 3 updates in the / directory: [redis](https://github.com/redis/redis-py), [pymongo](https://github.com/mongodb/mongo-python-driver) and [h5py](https://github.com/h5py/h5py). Updates `redis` from 5.0.8 to 5.1.0 - [Release notes](https://github.com/redis/redis-py/releases) - [Changelog](https://github.com/redis/redis-py/blob/master/CHANGES) - [Commits](https://github.com/redis/redis-py/compare/v5.0.8...v5.1.0) Updates `pymongo` from 4.8.0 to 4.10.0 - [Release notes](https://github.com/mongodb/mongo-python-driver/releases) - [Changelog](https://github.com/mongodb/mongo-python-driver/blob/master/doc/changelog.rst) - [Commits](https://github.com/mongodb/mongo-python-driver/compare/4.8.0...4.10.0) Updates `h5py` from 3.11.0 to 3.12.1 - [Release notes](https://github.com/h5py/h5py/releases) - [Changelog](https://github.com/h5py/h5py/blob/master/docs/release_guide.rst) - [Commits](https://github.com/h5py/h5py/compare/3.11.0...3.12.1) --- updated-dependencies: - dependency-name: redis dependency-type: direct:development update-type: version-update:semver-minor dependency-group: requirements - dependency-name: pymongo dependency-type: direct:development update-type: version-update:semver-minor dependency-group: requirements - dependency-name: h5py dependency-type: direct:development update-type: version-update:semver-minor dependency-group: requirements ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- requirements_dev_optional.txt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/requirements_dev_optional.txt b/requirements_dev_optional.txt index 9d75d78e44..16a4bd868b 100644 --- a/requirements_dev_optional.txt +++ b/requirements_dev_optional.txt @@ -8,16 +8,16 @@ ipywidgets==8.1.5 # don't let pyup change pinning for azure-storage-blob, need to pin to older # version to get compatibility with azure storage emulator on appveyor (FIXME) azure-storage-blob==12.21.0 # pyup: ignore -redis==5.0.8 +redis==5.1.0 types-redis types-setuptools -pymongo==4.8.0 +pymongo==4.10.0 # optional test requirements coverage pytest-cov==5.0.0 pytest-doctestplus==1.2.1 pytest-timeout==2.3.1 -h5py==3.11.0 +h5py==3.12.1 fsspec==2023.12.2 s3fs==2023.12.2 moto[server]>=5.0.1 From 559c2e3e34b0e5f3d408083045baf3e9f7cc0cd7 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 2 Oct 2024 08:39:24 +0200 Subject: [PATCH 126/130] Bump pymongo from 4.10.0 to 4.10.1 in the requirements group (#2288) Bumps the requirements group with 1 update: [pymongo](https://github.com/mongodb/mongo-python-driver). Updates `pymongo` from 4.10.0 to 4.10.1 - [Release notes](https://github.com/mongodb/mongo-python-driver/releases) - [Changelog](https://github.com/mongodb/mongo-python-driver/blob/master/doc/changelog.rst) - [Commits](https://github.com/mongodb/mongo-python-driver/compare/4.10.0...4.10.1) --- updated-dependencies: - dependency-name: pymongo dependency-type: direct:development update-type: version-update:semver-patch dependency-group: requirements ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- requirements_dev_optional.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements_dev_optional.txt b/requirements_dev_optional.txt index 16a4bd868b..52c4a737a1 100644 --- a/requirements_dev_optional.txt +++ b/requirements_dev_optional.txt @@ -11,7 +11,7 @@ azure-storage-blob==12.21.0 # pyup: ignore redis==5.1.0 types-redis types-setuptools -pymongo==4.10.0 +pymongo==4.10.1 # optional test requirements coverage pytest-cov==5.0.0 From 6b9ab9e29f526986c5c7519dd4063546d5a81d14 Mon Sep 17 00:00:00 2001 From: David Stansby Date: Wed, 2 Oct 2024 16:30:35 +0200 Subject: [PATCH 127/130] Fix low contrast box titles (#2287) (#2292) --- docs/_static/custom.css | 2 -- 1 file changed, 2 deletions(-) diff --git a/docs/_static/custom.css b/docs/_static/custom.css index 87dd70e347..4e664a596b 100644 --- a/docs/_static/custom.css +++ b/docs/_static/custom.css @@ -74,7 +74,6 @@ Nat Methods 8, 441 (2011). https://doi.org/10.1038/nmeth.1618 .sd-card .sd-card-header { border: none; background-color: white; - color: #150458 !important; font-size: var(--pst-font-size-h5); font-weight: bold; padding: 2.5rem 0rem 0.5rem 0rem; @@ -107,7 +106,6 @@ html[data-theme=dark] .sd-shadow-sm { html[data-theme=dark] .sd-card .sd-card-header { background-color:var(--pst-color-background); - color: #150458 !important; } html[data-theme=dark] .sd-card .sd-card-footer { From f175de475fd2d45e9df90ad313e0e57eb7ef7fa2 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 7 Oct 2024 10:35:58 -0700 Subject: [PATCH 128/130] Bump pypa/gh-action-pypi-publish in the actions group (#2304) Bumps the actions group with 1 update: [pypa/gh-action-pypi-publish](https://github.com/pypa/gh-action-pypi-publish). Updates `pypa/gh-action-pypi-publish` from 1.10.2 to 1.10.3 - [Release notes](https://github.com/pypa/gh-action-pypi-publish/releases) - [Commits](https://github.com/pypa/gh-action-pypi-publish/compare/v1.10.2...v1.10.3) --- updated-dependencies: - dependency-name: pypa/gh-action-pypi-publish dependency-type: direct:production update-type: version-update:semver-patch dependency-group: actions ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/releases.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/releases.yml b/.github/workflows/releases.yml index dccfbc208f..26b669abea 100644 --- a/.github/workflows/releases.yml +++ b/.github/workflows/releases.yml @@ -64,7 +64,7 @@ jobs: with: name: releases path: dist - - uses: pypa/gh-action-pypi-publish@v1.10.2 + - uses: pypa/gh-action-pypi-publish@v1.10.3 with: user: __token__ password: ${{ secrets.pypi_password }} From a1dfd016983733277ed9a925aaef6fb2a08938e5 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 7 Oct 2024 15:21:28 -0700 Subject: [PATCH 129/130] Bump the requirements group with 2 updates (#2303) Bumps the requirements group with 2 updates: [numpy](https://github.com/numpy/numpy) and [redis](https://github.com/redis/redis-py). Updates `numpy` from 2.1.1 to 2.1.2 - [Release notes](https://github.com/numpy/numpy/releases) - [Changelog](https://github.com/numpy/numpy/blob/main/doc/RELEASE_WALKTHROUGH.rst) - [Commits](https://github.com/numpy/numpy/compare/v2.1.1...v2.1.2) Updates `redis` from 5.1.0 to 5.1.1 - [Release notes](https://github.com/redis/redis-py/releases) - [Changelog](https://github.com/redis/redis-py/blob/master/CHANGES) - [Commits](https://github.com/redis/redis-py/compare/v5.1.0...v5.1.1) --- updated-dependencies: - dependency-name: numpy dependency-type: direct:development update-type: version-update:semver-patch dependency-group: requirements - dependency-name: redis dependency-type: direct:development update-type: version-update:semver-patch dependency-group: requirements ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- requirements_dev_numpy.txt | 2 +- requirements_dev_optional.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements_dev_numpy.txt b/requirements_dev_numpy.txt index 190610d3d3..4a619aa3ef 100644 --- a/requirements_dev_numpy.txt +++ b/requirements_dev_numpy.txt @@ -1,4 +1,4 @@ # Break this out into a separate file to allow testing against # different versions of numpy. This file should pin to the latest # numpy version. -numpy==2.1.1 +numpy==2.1.2 diff --git a/requirements_dev_optional.txt b/requirements_dev_optional.txt index 52c4a737a1..df1d4fd793 100644 --- a/requirements_dev_optional.txt +++ b/requirements_dev_optional.txt @@ -8,7 +8,7 @@ ipywidgets==8.1.5 # don't let pyup change pinning for azure-storage-blob, need to pin to older # version to get compatibility with azure storage emulator on appveyor (FIXME) azure-storage-blob==12.21.0 # pyup: ignore -redis==5.1.0 +redis==5.1.1 types-redis types-setuptools pymongo==4.10.1 From eaf5d7ab4c5a791ce112e0ced45bca2288eb9d69 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 10 Oct 2024 08:30:48 +0100 Subject: [PATCH 130/130] Bump numcodecs from 0.13.0 to 0.13.1 in the requirements group (#2326) Bumps the requirements group with 1 update: [numcodecs](https://github.com/zarr-developers/numcodecs). Updates `numcodecs` from 0.13.0 to 0.13.1 - [Release notes](https://github.com/zarr-developers/numcodecs/releases) - [Changelog](https://github.com/zarr-developers/numcodecs/blob/main/docs/release.rst) - [Commits](https://github.com/zarr-developers/numcodecs/compare/v0.13.0...v0.13.1) --- updated-dependencies: - dependency-name: numcodecs dependency-type: direct:development update-type: version-update:semver-patch dependency-group: requirements ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- requirements_dev_minimal.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements_dev_minimal.txt b/requirements_dev_minimal.txt index 1d29404d6c..caa078cc82 100644 --- a/requirements_dev_minimal.txt +++ b/requirements_dev_minimal.txt @@ -1,7 +1,7 @@ # library requirements asciitree==0.3.3 fasteners==0.19 -numcodecs==0.13.0 +numcodecs==0.13.1 msgpack-python==0.5.6 setuptools-scm==8.1.0 # test requirements