From ee148ae3014f581e48209a0609c7d8f17a148cc9 Mon Sep 17 00:00:00 2001 From: fynnbe Date: Wed, 8 May 2024 14:11:08 +0200 Subject: [PATCH 1/5] test and fix interprete_file_source --- bioimageio/spec/_internal/io.py | 4 ++-- tests/test_internal/test_io.py | 31 +++++++++++++++++++++++++++++-- 2 files changed, 31 insertions(+), 4 deletions(-) diff --git a/bioimageio/spec/_internal/io.py b/bioimageio/spec/_internal/io.py index ca7066cda..09378d65f 100644 --- a/bioimageio/spec/_internal/io.py +++ b/bioimageio/spec/_internal/io.py @@ -67,7 +67,7 @@ from .._internal.root_url import RootHttpUrl from .._internal.url import HttpUrl from .._internal.validated_string import ValidatedString -from .._internal.validation_context import ValidationContext, validation_context_var +from .._internal.validation_context import validation_context_var from .validator_annotations import AfterValidator if sys.version_info < (3, 10): @@ -527,7 +527,7 @@ def interprete_file_source(file_source: PermissiveFileSource) -> StrictFileSourc if isinstance(file_source, pydantic.AnyUrl): file_source = str(file_source) - with ValidationContext(perform_io_checks=False): + with validation_context_var.get().replace(perform_io_checks=False): strict = _strict_file_source_adapter.validate_python(file_source) return strict diff --git a/tests/test_internal/test_io.py b/tests/test_internal/test_io.py index 0ba4193e1..e95e4881b 100644 --- a/tests/test_internal/test_io.py +++ b/tests/test_internal/test_io.py @@ -1,10 +1,11 @@ -from pathlib import PurePath +from pathlib import Path, PurePath from typing import Any import pytest from pydantic import ValidationError -from bioimageio.spec._internal.validation_context import ValidationContext +from bioimageio.spec import ValidationContext +from bioimageio.spec.common import RelativeFilePath @pytest.mark.parametrize("p", [PurePath("maybe_a_file"), "maybe_a_file"]) @@ -43,3 +44,29 @@ def test_invalid_relative_file_path_io_check(p: Any): with ValidationContext(perform_io_checks=True), pytest.raises(ValidationError): _ = RelativeFilePath(p) + + +def test_interprete_file_source_from_str(): + from bioimageio.spec._internal.io import interprete_file_source + + src = f"{PurePath(__file__).parent.name}/{PurePath(__file__).name}" + + with ValidationContext(root=Path(__file__).parent.parent): + interpreted = interprete_file_source(src) + assert isinstance(interpreted, RelativeFilePath) + assert isinstance(interpreted.absolute, Path) + assert interpreted.absolute.exists() + + +def test_interprete_file_source_from_rel_path(): + from bioimageio.spec._internal.io import interprete_file_source + + with ValidationContext(root=Path(__file__).parent.parent): + src = RelativeFilePath( + PurePath(PurePath(__file__).parent.name) / PurePath(__file__).name + ) + + interpreted = interprete_file_source(src) + assert isinstance(interpreted, RelativeFilePath) + assert isinstance(interpreted.absolute, Path) + assert interpreted.absolute.exists() From 26711808134ac60690b5b6df726d90af824befb3 Mon Sep 17 00:00:00 2001 From: fynnbe Date: Wed, 8 May 2024 14:12:38 +0200 Subject: [PATCH 2/5] bump post --- README.md | 3 ++- bioimageio/spec/VERSION | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index f4f893856..55b14b4f1 100644 --- a/README.md +++ b/README.md @@ -89,9 +89,10 @@ Made with [contrib.rocks](https://contrib.rocks). ### bioimageio.spec Python package -#### bioimageio.spec 0.5.2post4 (to be released) +#### bioimageio.spec 0.5.2post4 * resolve backup DOIs +* fix resolving relative file paths given as strings #### bioimageio.spec 0.5.2post3 diff --git a/bioimageio/spec/VERSION b/bioimageio/spec/VERSION index d7a969a89..50f0d5c70 100644 --- a/bioimageio/spec/VERSION +++ b/bioimageio/spec/VERSION @@ -1,3 +1,3 @@ { - "version": "0.5.2post3" + "version": "0.5.2post4" } From e30d48e3f78bdd92f6ceddd87920d75654dbc605 Mon Sep 17 00:00:00 2001 From: fynnbe Date: Wed, 8 May 2024 15:12:00 +0200 Subject: [PATCH 3/5] add known_files to ValidationContext and use them in sha validation --- README.md | 1 + bioimageio/spec/_internal/field_warning.py | 4 +- bioimageio/spec/_internal/io.py | 38 +++++++------------ bioimageio/spec/_internal/io_basics.py | 18 ++++++++- bioimageio/spec/_internal/io_utils.py | 3 +- bioimageio/spec/_internal/types.py | 2 +- .../spec/_internal/validation_context.py | 8 +++- bioimageio/spec/application/v0_3.py | 2 +- bioimageio/spec/collection/v0_3.py | 2 +- bioimageio/spec/common.py | 2 +- bioimageio/spec/dataset/v0_3.py | 2 +- bioimageio/spec/generic/v0_3.py | 2 +- bioimageio/spec/model/v0_4.py | 2 +- bioimageio/spec/model/v0_5.py | 2 +- bioimageio/spec/notebook/v0_3.py | 2 +- tests/test_internal/test_types.py | 3 +- 16 files changed, 51 insertions(+), 42 deletions(-) diff --git a/README.md b/README.md index 55b14b4f1..28913f787 100644 --- a/README.md +++ b/README.md @@ -93,6 +93,7 @@ Made with [contrib.rocks](https://contrib.rocks). * resolve backup DOIs * fix resolving relative file paths given as strings +* allow to bypass download and hashing of known files #### bioimageio.spec 0.5.2post3 diff --git a/bioimageio/spec/_internal/field_warning.py b/bioimageio/spec/_internal/field_warning.py index 9e4e0443e..65d995e81 100644 --- a/bioimageio/spec/_internal/field_warning.py +++ b/bioimageio/spec/_internal/field_warning.py @@ -15,8 +15,8 @@ ) from typing_extensions import Annotated, LiteralString -from .._internal.validation_context import validation_context_var -from .._internal.warning_levels import WARNING, WarningSeverity +from .validation_context import validation_context_var +from .warning_levels import WARNING, WarningSeverity if TYPE_CHECKING: from pydantic.functional_validators import _V2Validator # type: ignore diff --git a/bioimageio/spec/_internal/io.py b/bioimageio/spec/_internal/io.py index 09378d65f..fdbb90082 100644 --- a/bioimageio/spec/_internal/io.py +++ b/bioimageio/spec/_internal/io.py @@ -10,7 +10,6 @@ from pathlib import Path, PurePath from typing import ( Any, - ClassVar, Dict, Generic, Iterable, @@ -38,7 +37,6 @@ PrivateAttr, RootModel, SerializationInfo, - StringConstraints, TypeAdapter, model_validator, ) @@ -53,21 +51,21 @@ ) from typing_extensions import TypeAliasType as _TypeAliasType -from .._internal._settings import settings -from .._internal.io_basics import ( +from ._settings import settings +from .io_basics import ( ALL_BIOIMAGEIO_YAML_NAMES, ALTERNATIVE_BIOIMAGEIO_YAML_NAMES, BIOIMAGEIO_YAML, AbsoluteDirectory, AbsoluteFilePath, FileName, + Sha256, ) -from .._internal.node import Node -from .._internal.packaging_context import packaging_context_var -from .._internal.root_url import RootHttpUrl -from .._internal.url import HttpUrl -from .._internal.validated_string import ValidatedString -from .._internal.validation_context import validation_context_var +from .node import Node +from .packaging_context import packaging_context_var +from .root_url import RootHttpUrl +from .url import HttpUrl +from .validation_context import validation_context_var from .validator_annotations import AfterValidator if sys.version_info < (3, 10): @@ -76,19 +74,6 @@ SLOTS = {"slots": True} -class Sha256(ValidatedString): - """SHA-256 hash value""" - - root_model: ClassVar[Type[RootModel[Any]]] = RootModel[ - Annotated[ - str, - StringConstraints( - strip_whitespace=True, to_lower=True, min_length=64, max_length=64 - ), - ] - ] - - AbsolutePathT = TypeVar( "AbsolutePathT", bound=Union[HttpUrl, AbsoluteDirectory, AbsoluteFilePath] ) @@ -617,9 +602,12 @@ def validate_sha256(self) -> Self: context = validation_context_var.get() if not context.perform_io_checks: return self + elif (src_str := str(self.source)) in context.known_files: + actual_sha = context.known_files[src_str] + else: + local_source = download(self.source, sha256=self.sha256).path + actual_sha = get_sha256(local_source) - local_source = download(self.source, sha256=self.sha256).path - actual_sha = get_sha256(local_source) if self.sha256 is None: self.sha256 = actual_sha elif self.sha256 != actual_sha: diff --git a/bioimageio/spec/_internal/io_basics.py b/bioimageio/spec/_internal/io_basics.py index 0c7c98169..312b353a9 100644 --- a/bioimageio/spec/_internal/io_basics.py +++ b/bioimageio/spec/_internal/io_basics.py @@ -1,9 +1,12 @@ from pathlib import Path +from typing import Any, ClassVar, Type from annotated_types import Predicate -from pydantic import DirectoryPath, FilePath +from pydantic import DirectoryPath, FilePath, RootModel, StringConstraints from typing_extensions import Annotated +from .validated_string import ValidatedString + FileName = str AbsoluteDirectory = Annotated[DirectoryPath, Predicate(Path.is_absolute)] AbsoluteFilePath = Annotated[FilePath, Predicate(Path.is_absolute)] @@ -11,3 +14,16 @@ BIOIMAGEIO_YAML = "bioimageio.yaml" ALTERNATIVE_BIOIMAGEIO_YAML_NAMES = ("rdf.yaml", "model.yaml") ALL_BIOIMAGEIO_YAML_NAMES = (BIOIMAGEIO_YAML,) + ALTERNATIVE_BIOIMAGEIO_YAML_NAMES + + +class Sha256(ValidatedString): + """SHA-256 hash value""" + + root_model: ClassVar[Type[RootModel[Any]]] = RootModel[ + Annotated[ + str, + StringConstraints( + strip_whitespace=True, to_lower=True, min_length=64, max_length=64 + ), + ] + ] diff --git a/bioimageio/spec/_internal/io_utils.py b/bioimageio/spec/_internal/io_utils.py index 64e60d896..0b1faac95 100644 --- a/bioimageio/spec/_internal/io_utils.py +++ b/bioimageio/spec/_internal/io_utils.py @@ -32,12 +32,11 @@ FileDescr, HashKwargs, OpenedBioimageioYaml, - Sha256, YamlValue, download, find_bioimageio_yaml_file_name, ) -from .io_basics import FileName +from .io_basics import FileName, Sha256 from .types import FileSource, PermissiveFileSource yaml = YAML(typ="safe") diff --git a/bioimageio/spec/_internal/types.py b/bioimageio/spec/_internal/types.py index 1d8a930da..bd1064f9a 100644 --- a/bioimageio/spec/_internal/types.py +++ b/bioimageio/spec/_internal/types.py @@ -14,10 +14,10 @@ from .io import ImportantFileSource as ImportantFileSource from .io import PermissiveFileSource as PermissiveFileSource from .io import RelativeFilePath as RelativeFilePath -from .io import Sha256 as Sha256 from .io_basics import AbsoluteDirectory as AbsoluteDirectory from .io_basics import AbsoluteFilePath as AbsoluteFilePath from .io_basics import FileName as FileName +from .io_basics import Sha256 as Sha256 from .license_id import DeprecatedLicenseId as DeprecatedLicenseId from .license_id import LicenseId as LicenseId from .url import HttpUrl as HttpUrl diff --git a/bioimageio/spec/_internal/validation_context.py b/bioimageio/spec/_internal/validation_context.py index 123177ffc..fb2db7239 100644 --- a/bioimageio/spec/_internal/validation_context.py +++ b/bioimageio/spec/_internal/validation_context.py @@ -3,13 +3,14 @@ from contextvars import ContextVar, Token from dataclasses import dataclass, field from pathlib import Path -from typing import List, Optional, Union +from types import MappingProxyType +from typing import List, Mapping, Optional, Union from urllib.parse import urlsplit, urlunsplit from pydantic import DirectoryPath from ._settings import settings -from .io_basics import AbsoluteDirectory, FileName +from .io_basics import AbsoluteDirectory, FileName, Sha256 from .root_url import RootHttpUrl from .warning_levels import WarningLevel @@ -38,6 +39,9 @@ class ValidationContext: Existence of local absolute file paths is still being checked.""" + known_files: Mapping[str, Sha256] = MappingProxyType({}) + """allows to bypass download and hashing of referenced files""" + def replace( self, root: Optional[Union[RootHttpUrl, DirectoryPath]] = None, diff --git a/bioimageio/spec/application/v0_3.py b/bioimageio/spec/application/v0_3.py index d6192e6be..aaff785ad 100644 --- a/bioimageio/spec/application/v0_3.py +++ b/bioimageio/spec/application/v0_3.py @@ -5,8 +5,8 @@ from .._internal.common_nodes import Node from .._internal.io import FileDescr as FileDescr -from .._internal.io import Sha256 as Sha256 from .._internal.io_basics import AbsoluteFilePath as AbsoluteFilePath +from .._internal.io_basics import Sha256 as Sha256 from .._internal.types import ImportantFileSource from .._internal.url import HttpUrl as HttpUrl from ..generic.v0_3 import VALID_COVER_IMAGE_EXTENSIONS as VALID_COVER_IMAGE_EXTENSIONS diff --git a/bioimageio/spec/collection/v0_3.py b/bioimageio/spec/collection/v0_3.py index 021440b7c..25f836094 100644 --- a/bioimageio/spec/collection/v0_3.py +++ b/bioimageio/spec/collection/v0_3.py @@ -19,9 +19,9 @@ from .._internal.common_nodes import InvalidDescr, Node from .._internal.field_warning import issue_warning from .._internal.io import FileDescr as FileDescr -from .._internal.io import Sha256 as Sha256 from .._internal.io import YamlValue from .._internal.io_basics import AbsoluteFilePath as AbsoluteFilePath +from .._internal.io_basics import Sha256 as Sha256 from .._internal.io_utils import open_bioimageio_yaml from .._internal.types import FileSource, NotEmpty from .._internal.url import HttpUrl as HttpUrl diff --git a/bioimageio/spec/common.py b/bioimageio/spec/common.py index e2a7f18ed..9ee7087b2 100644 --- a/bioimageio/spec/common.py +++ b/bioimageio/spec/common.py @@ -4,11 +4,11 @@ from ._internal.io import BioimageioYamlContent as BioimageioYamlContent from ._internal.io import BioimageioYamlSource as BioimageioYamlSource from ._internal.io import FileDescr as FileDescr -from ._internal.io import Sha256 as Sha256 from ._internal.io import YamlValue as YamlValue from ._internal.io_basics import AbsoluteDirectory as AbsoluteDirectory from ._internal.io_basics import AbsoluteFilePath as AbsoluteFilePath from ._internal.io_basics import FileName as FileName +from ._internal.io_basics import Sha256 as Sha256 from ._internal.root_url import RootHttpUrl as RootHttpUrl from ._internal.types import FileSource as FileSource from ._internal.types import PermissiveFileSource as PermissiveFileSource diff --git a/bioimageio/spec/dataset/v0_3.py b/bioimageio/spec/dataset/v0_3.py index cd0fa13d3..241d593d6 100644 --- a/bioimageio/spec/dataset/v0_3.py +++ b/bioimageio/spec/dataset/v0_3.py @@ -4,8 +4,8 @@ from .._internal.common_nodes import InvalidDescr, Node from .._internal.io import FileDescr as FileDescr -from .._internal.io import Sha256 as Sha256 from .._internal.io_basics import AbsoluteFilePath as AbsoluteFilePath +from .._internal.io_basics import Sha256 as Sha256 from .._internal.url import HttpUrl as HttpUrl from ..generic.v0_3 import VALID_COVER_IMAGE_EXTENSIONS as VALID_COVER_IMAGE_EXTENSIONS from ..generic.v0_3 import Author as Author diff --git a/bioimageio/spec/generic/v0_3.py b/bioimageio/spec/generic/v0_3.py index b770ad7ce..0fc9cbc26 100644 --- a/bioimageio/spec/generic/v0_3.py +++ b/bioimageio/spec/generic/v0_3.py @@ -38,8 +38,8 @@ validate_suffix, ) from .._internal.io import FileDescr as FileDescr -from .._internal.io import Sha256 as Sha256 from .._internal.io_basics import AbsoluteFilePath +from .._internal.io_basics import Sha256 as Sha256 from .._internal.license_id import LicenseId from .._internal.types import ( DeprecatedLicenseId, diff --git a/bioimageio/spec/model/v0_4.py b/bioimageio/spec/model/v0_4.py index a47f11ae4..33845f2e3 100644 --- a/bioimageio/spec/model/v0_4.py +++ b/bioimageio/spec/model/v0_4.py @@ -47,8 +47,8 @@ include_in_package_serializer, ) from .._internal.io import FileDescr as FileDescr -from .._internal.io import Sha256 as Sha256 from .._internal.io_basics import AbsoluteFilePath as AbsoluteFilePath +from .._internal.io_basics import Sha256 as Sha256 from .._internal.packaging_context import packaging_context_var from .._internal.types import Datetime as Datetime from .._internal.types import Identifier as Identifier diff --git a/bioimageio/spec/model/v0_5.py b/bioimageio/spec/model/v0_5.py index 4abe11626..05da86f46 100644 --- a/bioimageio/spec/model/v0_5.py +++ b/bioimageio/spec/model/v0_5.py @@ -59,9 +59,9 @@ from .._internal.field_warning import issue_warning, warn from .._internal.io import BioimageioYamlContent as BioimageioYamlContent from .._internal.io import FileDescr as FileDescr -from .._internal.io import Sha256 as Sha256 from .._internal.io import WithSuffix, YamlValue, download from .._internal.io_basics import AbsoluteFilePath as AbsoluteFilePath +from .._internal.io_basics import Sha256 as Sha256 from .._internal.io_utils import load_array from .._internal.types import Datetime as Datetime from .._internal.types import DeprecatedLicenseId as DeprecatedLicenseId diff --git a/bioimageio/spec/notebook/v0_3.py b/bioimageio/spec/notebook/v0_3.py index ddefe1523..b786c218d 100644 --- a/bioimageio/spec/notebook/v0_3.py +++ b/bioimageio/spec/notebook/v0_3.py @@ -2,8 +2,8 @@ from .._internal.common_nodes import Node from .._internal.io import FileDescr as FileDescr -from .._internal.io import Sha256 as Sha256 from .._internal.io_basics import AbsoluteFilePath as AbsoluteFilePath +from .._internal.io_basics import Sha256 as Sha256 from .._internal.url import HttpUrl as HttpUrl from ..generic.v0_3 import VALID_COVER_IMAGE_EXTENSIONS as VALID_COVER_IMAGE_EXTENSIONS from ..generic.v0_3 import Author as Author diff --git a/tests/test_internal/test_types.py b/tests/test_internal/test_types.py index 0a8a74410..3cfff996d 100644 --- a/tests/test_internal/test_types.py +++ b/tests/test_internal/test_types.py @@ -6,6 +6,7 @@ from pydantic import PlainSerializer, TypeAdapter from typing_extensions import Annotated +import bioimageio.spec._internal.io_basics from bioimageio.spec._internal import types from bioimageio.spec._internal.io import RelativeFilePath from bioimageio.spec._internal.types import Datetime, SiUnit @@ -30,7 +31,7 @@ types.FileName: "lala.py", types.Version: "1.0", types.HttpUrl: "http://example.com", - types.Sha256: "0" * 64, + bioimageio.spec._internal.io_basics.Sha256: "0" * 64, } IGNORE_TYPES_MEMBERS = { From cc744b2154e452549e6adac6b139e0410a0e3158 Mon Sep 17 00:00:00 2001 From: fynnbe Date: Wed, 8 May 2024 15:21:34 +0200 Subject: [PATCH 4/5] cache known_files --- bioimageio/spec/_internal/io.py | 3 +++ bioimageio/spec/_internal/validation_context.py | 7 ++++--- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/bioimageio/spec/_internal/io.py b/bioimageio/spec/_internal/io.py index fdbb90082..526116023 100644 --- a/bioimageio/spec/_internal/io.py +++ b/bioimageio/spec/_internal/io.py @@ -7,6 +7,7 @@ from dataclasses import dataclass from datetime import date as _date from datetime import datetime as _datetime +from functools import lru_cache from pathlib import Path, PurePath from typing import ( Any, @@ -607,6 +608,7 @@ def validate_sha256(self) -> Self: else: local_source = download(self.source, sha256=self.sha256).path actual_sha = get_sha256(local_source) + context.known_files[str(self.source)] = actual_sha if self.sha256 is None: self.sha256 = actual_sha @@ -644,6 +646,7 @@ def extract_file_name( return url.path.split("/")[-1] +@lru_cache def get_sha256(path: Path) -> Sha256: """from https://stackoverflow.com/a/44873382""" h = hashlib.sha256() diff --git a/bioimageio/spec/_internal/validation_context.py b/bioimageio/spec/_internal/validation_context.py index fb2db7239..f2f652fe0 100644 --- a/bioimageio/spec/_internal/validation_context.py +++ b/bioimageio/spec/_internal/validation_context.py @@ -3,8 +3,7 @@ from contextvars import ContextVar, Token from dataclasses import dataclass, field from pathlib import Path -from types import MappingProxyType -from typing import List, Mapping, Optional, Union +from typing import Dict, List, Optional, Union from urllib.parse import urlsplit, urlunsplit from pydantic import DirectoryPath @@ -39,7 +38,7 @@ class ValidationContext: Existence of local absolute file paths is still being checked.""" - known_files: Mapping[str, Sha256] = MappingProxyType({}) + known_files: Dict[str, Sha256] = field(default_factory=dict) """allows to bypass download and hashing of referenced files""" def replace( @@ -49,6 +48,7 @@ def replace( log_warnings: Optional[bool] = None, file_name: Optional[str] = None, perform_io_checks: Optional[bool] = None, + known_files: Optional[Dict[str, Sha256]] = None, ) -> "ValidationContext": return ValidationContext( root=self.root if root is None else root, @@ -62,6 +62,7 @@ def replace( if perform_io_checks is None else perform_io_checks ), + known_files=self.known_files if known_files is None else known_files, ) def __enter__(self): From a5b545a0be18370817234e26a068324f06dde2d4 Mon Sep 17 00:00:00 2001 From: fynnbe Date: Wed, 8 May 2024 15:50:16 +0200 Subject: [PATCH 5/5] add and fix test_known_files --- bioimageio/spec/_internal/io.py | 8 +++++--- bioimageio/spec/_internal/url.py | 3 ++- tests/test_internal/test_io.py | 32 ++++++++++++++++++++++++++++++++ 3 files changed, 39 insertions(+), 4 deletions(-) diff --git a/bioimageio/spec/_internal/io.py b/bioimageio/spec/_internal/io.py index 526116023..73857c0a1 100644 --- a/bioimageio/spec/_internal/io.py +++ b/bioimageio/spec/_internal/io.py @@ -174,7 +174,8 @@ def get_absolute( absolute = self._get_absolute_impl(root) if ( isinstance(absolute, Path) - and validation_context_var.get().perform_io_checks + and (context := validation_context_var.get()).perform_io_checks + and str(self.root) not in context.known_files and not absolute.exists() ): raise ValueError(f"{absolute} does not exist") @@ -195,7 +196,8 @@ def get_absolute( absolute = self._get_absolute_impl(root) if ( isinstance(absolute, Path) - and validation_context_var.get().perform_io_checks + and (context := validation_context_var.get()).perform_io_checks + and str(self.root) not in context.known_files and not absolute.is_file() ): raise ValueError(f"{absolute} does not point to an existing file") @@ -221,7 +223,7 @@ def get_absolute( FileSource = Annotated[ - Union[FilePath, RelativeFilePath, HttpUrl, pydantic.HttpUrl], + Union[FilePath, HttpUrl, RelativeFilePath, pydantic.HttpUrl], Field(union_mode="left_to_right"), ] PermissiveFileSource = Union[FileSource, str] diff --git a/bioimageio/spec/_internal/url.py b/bioimageio/spec/_internal/url.py index c63b19c76..557c31913 100644 --- a/bioimageio/spec/_internal/url.py +++ b/bioimageio/spec/_internal/url.py @@ -13,7 +13,8 @@ def _validate_url(url: Union[str, pydantic.HttpUrl]) -> pydantic.AnyUrl: url = str(url) - if not validation_context_var.get().perform_io_checks: + context = validation_context_var.get() + if not context.perform_io_checks or url in context.known_files: return pydantic.AnyUrl(url) val_url = url diff --git a/tests/test_internal/test_io.py b/tests/test_internal/test_io.py index e95e4881b..2335da45e 100644 --- a/tests/test_internal/test_io.py +++ b/tests/test_internal/test_io.py @@ -70,3 +70,35 @@ def test_interprete_file_source_from_rel_path(): assert isinstance(interpreted, RelativeFilePath) assert isinstance(interpreted.absolute, Path) assert interpreted.absolute.exists() + + +def test_known_files(tmp_path: Path): + from bioimageio.spec._internal.io import FileDescr, get_sha256 + + file_name = "lala.txt" + src = tmp_path / file_name + _ = src.write_text("lala") + sha = get_sha256(src) + + with ValidationContext(root=tmp_path): + # set sha on loading + file_descr = FileDescr(source=file_name) # pyright: ignore[reportArgumentType] + assert file_descr.sha256 == sha + + # validate given sha + file_descr = FileDescr( + source=file_name, sha256=sha # pyright: ignore[reportArgumentType] + ) + assert file_descr.sha256 == sha + + # give known files to bypass file io + with ValidationContext(known_files={file_name: sha}): + # set sha on loading + file_descr = FileDescr(source=file_name) # pyright: ignore[reportArgumentType] + assert file_descr.sha256 == sha + + # validate given sha + file_descr = FileDescr( + source=file_name, sha256=sha # pyright: ignore[reportArgumentType] + ) + assert file_descr.sha256 == sha