Wire up Zarr property tests EAR-1189 (#68)

* Wire up Zarr property tests

* Cleanup

* fix hypothesis cache

* Fix hypothesis CI

* temp

* update

* Silence actionlint

* silence ruff

* updates

* lint

* WIP

* easy workflow lint

* sketchyworkflowlint

* Update icechunk-python/tests/test_zarr/test_properties.py

.github/workflows/deploy-docs.yaml

@@ -19,7 +19,8 @@ jobs:
           git config user.name github-actions[bot]
           git config user.email 41898282+github-actions[bot]@users.noreply.github.com
 
-      - uses: actions/setup-python@v5
+      - id: setup-python
+        uses: actions/setup-python@v5
         with:
           python-version: 3.x
 

.github/workflows/publish-rust-library.yml

@@ -24,7 +24,7 @@ jobs:
           docker compose up -d minio
       - name: Wait for MinIO to be ready
         run: |
-          for i in {1..10}; do
+          for _ in {1..10}; do
               if curl --silent --fail http://minio:9000/minio/health/live; then
               break
               fi
@@ -51,7 +51,7 @@ jobs:
         run: cargo install --locked cargo-deny
 
       - name: Check
-        if: matrix.os == 'ubuntu-latest' || github.event_name == 'push'
+        if: github.event_name == 'push'
         run: |
           just pre-commit
 

.github/workflows/python-check.yaml

@@ -35,7 +35,7 @@ jobs:
 
       - name: Wait for MinIO to be ready
         run: |
-          for i in {1..10}; do
+          for _ in {1..10}; do
               if curl --silent --fail http://minio:9000/minio/health/live; then
               break
               fi
@@ -50,10 +50,10 @@ jobs:
         uses: PyO3/maturin-action@v1
         with:
           working-directory: icechunk-python
-          target: ${{ matrix.platform.target }}
+          # target: ${{ matrix.platform.target }}
           args: --release --out dist --find-interpreter
           sccache: 'true'
-          manylinux: ${{ matrix.platform.manylinux }} # https://github.com/PyO3/maturin-action/issues/245
+          # manylinux: ${{ matrix.platform.manylinux }} # https://github.com/PyO3/maturin-action/issues/245
 
       - name: Cargo lint + format
         shell: bash
@@ -81,6 +81,15 @@ jobs:
           pip install icechunk['test'] --find-links dist --force-reinstall
           ruff check
 
+      - name: Restore cached hypothesis directory
+        id: restore-hypothesis-cache
+        uses: actions/cache/restore@v4
+        with:
+          path: icechunk-python/.hypothesis/
+          key: cache-hypothesis-${{ runner.os }}-${{ github.run_id }}
+          restore-keys: |
+            cache-hypothesis-
+
       - name: pytest
         shell: bash
         working-directory: icechunk-python
@@ -90,3 +99,12 @@ jobs:
           source .venv/bin/activate
           pip install icechunk['test'] --find-links dist --force-reinstall
           pytest
+
+      # explicitly save the cache so it gets updated, also do this even if it fails.
+      - name: Save cached hypothesis directory
+        id: save-hypothesis-cache
+        if: always()
+        uses: actions/cache/save@v4
+        with:
+          path: icechunk-python/.hypothesis/
+          key: cache-hypothesis-${{ runner.os }}-${{ github.run_id }}

.github/workflows/python-ci.yaml

@@ -46,7 +46,7 @@ jobs:
           docker compose up -d minio
       - name: Wait for MinIO to be ready
         run: |
-          for i in {1..10}; do
+          for _ in {1..10}; do
               if curl --silent --fail http://minio:9000/minio/health/live; then
               break
               fi

.github/workflows/rust-ci.yaml

@@ -57,7 +57,7 @@ jobs:
 
       - name: Wait for MinIO to be ready
         run: |
-          for i in {1..10}; do
+          for _ in {1..10}; do
               if curl --silent --fail http://minio:9000/minio/health/live; then
               break
               fi

.pre-commit-config.yaml

@@ -4,7 +4,8 @@ repos:
     hooks:
       - id: trailing-whitespace
       - id: end-of-file-fixer
-      # - id: check-yaml
+      - id: check-yaml
+        exclude: docs/mkdocs.yml
       - id: debug-statements
       - id: mixed-line-ending
   - repo: https://github.com/astral-sh/ruff-pre-commit
@@ -14,6 +15,17 @@ repos:
       - id: ruff-format
       - id: ruff
         args: ["--fix", "--show-fixes", "icechunk-python/"]
+  - repo: https://github.com/rhysd/actionlint
+    rev: v1.6.26
+    hooks:
+      - id: actionlint
+        files: ".github/workflows/"
+        args: ["-ignore", "SC1090", "-ignore", "SC2046", "-ignore", "SC2086", "-ignore", "SC2129", "-ignore", "SC2155"]
+  - repo: https://github.com/codespell-project/codespell
+    # Configuration for codespell is in .codespellrc
+    rev: v2.3.0
+    hooks:
+      - id: codespell
   - repo: local
     hooks:
       - id: format-lint
@@ -22,10 +34,5 @@ repos:
         entry: just pre-commit
         language: system
         pass_filenames: false
-  - repo: https://github.com/codespell-project/codespell
-    # Configuration for codespell is in .codespellrc
-    rev: v2.3.0
-    hooks:
-      - id: codespell
 
 exclude: 'tests/data/.*'

icechunk-python/pyproject.toml

@@ -37,6 +37,7 @@ test = [
   "ruff",
   "dask",
   "distributed",
+  "hypothesis",
 ]
 
 [tool.maturin]

icechunk-python/tests/test_zarr/test_properties.py

@@ -0,0 +1,51 @@
+import numpy as np
+import pytest
+from numpy.testing import assert_array_equal
+
+from icechunk import IcechunkStore, StorageConfig
+
+pytest.importorskip("hypothesis")
+
+import hypothesis.strategies as st
+from hypothesis import assume, given, settings
+
+from zarr.testing.strategies import arrays, numpy_arrays
+
+readable_characters = st.characters(
+    # only use characters within the "Latin Extended-A" subset of unicode
+    categories=["L", "N", "P"],
+    max_codepoint=0x017F,
+)
+_attr_keys = st.text(readable_characters, min_size=1)
+_attr_values = st.recursive(
+    st.none() | st.booleans() | st.text(readable_characters, min_size=1, max_size=5),
+    lambda children: st.lists(children) | st.dictionaries(_attr_keys, children),
+    max_leaves=3,
+)
+simple_attrs = st.none()
+
+icechunk_stores = st.builds(
+    IcechunkStore.create,
+    storage=st.builds(StorageConfig.memory, prefix=st.just("prefix")),
+    read_only=st.just(False),
+)
+
+
+@settings(report_multiple_bugs=True, deadline=None)
+@given(data=st.data(), nparray=numpy_arrays(zarr_formats=st.just(3)))
+def test_roundtrip(data: st.DataObject, nparray) -> None:
+    # TODO: support size-0 arrays GH392
+    assume(nparray.size > 0)
+    # TODO: fix complex fill values GH391
+    assume(not np.iscomplexobj(nparray))
+
+    zarray = data.draw(
+        arrays(
+            stores=icechunk_stores,
+            arrays=st.just(nparray),
+            zarr_formats=st.just(3),
+            # TODO: use all attrs once fixed GH393
+            attrs=simple_attrs,
+        )
+    )
+    assert_array_equal(nparray, zarray[:])