Postrelease

README.md

@@ -107,9 +107,11 @@ Made with [contrib.rocks](https://contrib.rocks).
 
 ### bioimageio.spec Python package
 
-#### bioimageio.spec 0.5.2post4 (to be released)
+#### bioimageio.spec 0.5.2post4
 
 * resolve backup DOIs
+* fix resolving relative file paths given as strings
+* allow to bypass download and hashing of known files
 
 #### bioimageio.spec 0.5.2post3
 

bioimageio/spec/VERSION

@@ -1,3 +1,3 @@
 {
-    "version": "0.5.2post3"
+    "version": "0.5.2post4"
 }

bioimageio/spec/_internal/field_warning.py

@@ -15,8 +15,8 @@
 )
 from typing_extensions import Annotated, LiteralString
 
-from .._internal.validation_context import validation_context_var
-from .._internal.warning_levels import WARNING, WarningSeverity
+from .validation_context import validation_context_var
+from .warning_levels import WARNING, WarningSeverity
 
 if TYPE_CHECKING:
     from pydantic.functional_validators import _V2Validator  # type: ignore

bioimageio/spec/_internal/io.py

@@ -7,10 +7,10 @@
 from dataclasses import dataclass
 from datetime import date as _date
 from datetime import datetime as _datetime
+from functools import lru_cache
 from pathlib import Path, PurePath
 from typing import (
     Any,
-    ClassVar,
     Dict,
     Generic,
     Iterable,
@@ -38,7 +38,6 @@
     PrivateAttr,
     RootModel,
     SerializationInfo,
-    StringConstraints,
     TypeAdapter,
     model_validator,
 )
@@ -53,21 +52,21 @@
 )
 from typing_extensions import TypeAliasType as _TypeAliasType
 
-from .._internal._settings import settings
-from .._internal.io_basics import (
+from ._settings import settings
+from .io_basics import (
     ALL_BIOIMAGEIO_YAML_NAMES,
     ALTERNATIVE_BIOIMAGEIO_YAML_NAMES,
     BIOIMAGEIO_YAML,
     AbsoluteDirectory,
     AbsoluteFilePath,
     FileName,
+    Sha256,
 )
-from .._internal.node import Node
-from .._internal.packaging_context import packaging_context_var
-from .._internal.root_url import RootHttpUrl
-from .._internal.url import HttpUrl
-from .._internal.validated_string import ValidatedString
-from .._internal.validation_context import ValidationContext, validation_context_var
+from .node import Node
+from .packaging_context import packaging_context_var
+from .root_url import RootHttpUrl
+from .url import HttpUrl
+from .validation_context import validation_context_var
 from .validator_annotations import AfterValidator
 
 if sys.version_info < (3, 10):
@@ -76,19 +75,6 @@
     SLOTS = {"slots": True}
 
 
-class Sha256(ValidatedString):
-    """SHA-256 hash value"""
-
-    root_model: ClassVar[Type[RootModel[Any]]] = RootModel[
-        Annotated[
-            str,
-            StringConstraints(
-                strip_whitespace=True, to_lower=True, min_length=64, max_length=64
-            ),
-        ]
-    ]
-
-
 AbsolutePathT = TypeVar(
     "AbsolutePathT", bound=Union[HttpUrl, AbsoluteDirectory, AbsoluteFilePath]
 )
@@ -188,7 +174,8 @@ def get_absolute(
         absolute = self._get_absolute_impl(root)
         if (
             isinstance(absolute, Path)
-            and validation_context_var.get().perform_io_checks
+            and (context := validation_context_var.get()).perform_io_checks
+            and str(self.root) not in context.known_files
             and not absolute.exists()
         ):
             raise ValueError(f"{absolute} does not exist")
@@ -209,7 +196,8 @@ def get_absolute(
         absolute = self._get_absolute_impl(root)
         if (
             isinstance(absolute, Path)
-            and validation_context_var.get().perform_io_checks
+            and (context := validation_context_var.get()).perform_io_checks
+            and str(self.root) not in context.known_files
             and not absolute.is_file()
         ):
             raise ValueError(f"{absolute} does not point to an existing file")
@@ -235,7 +223,7 @@ def get_absolute(
 
 
 FileSource = Annotated[
-    Union[FilePath, RelativeFilePath, HttpUrl, pydantic.HttpUrl],
+    Union[FilePath, HttpUrl, RelativeFilePath, pydantic.HttpUrl],
     Field(union_mode="left_to_right"),
 ]
 PermissiveFileSource = Union[FileSource, str]
@@ -527,7 +515,7 @@ def interprete_file_source(file_source: PermissiveFileSource) -> StrictFileSourc
     if isinstance(file_source, pydantic.AnyUrl):
         file_source = str(file_source)
 
-    with ValidationContext(perform_io_checks=False):
+    with validation_context_var.get().replace(perform_io_checks=False):
         strict = _strict_file_source_adapter.validate_python(file_source)
 
     return strict
@@ -617,9 +605,13 @@ def validate_sha256(self) -> Self:
         context = validation_context_var.get()
         if not context.perform_io_checks:
             return self
+        elif (src_str := str(self.source)) in context.known_files:
+            actual_sha = context.known_files[src_str]
+        else:
+            local_source = download(self.source, sha256=self.sha256).path
+            actual_sha = get_sha256(local_source)
+            context.known_files[str(self.source)] = actual_sha
 
-        local_source = download(self.source, sha256=self.sha256).path
-        actual_sha = get_sha256(local_source)
         if self.sha256 is None:
             self.sha256 = actual_sha
         elif self.sha256 != actual_sha:
@@ -656,6 +648,7 @@ def extract_file_name(
             return url.path.split("/")[-1]
 
 
+@lru_cache
 def get_sha256(path: Path) -> Sha256:
     """from https://stackoverflow.com/a/44873382"""
     h = hashlib.sha256()

bioimageio/spec/_internal/io_basics.py

@@ -1,13 +1,29 @@
 from pathlib import Path
+from typing import Any, ClassVar, Type
 
 from annotated_types import Predicate
-from pydantic import DirectoryPath, FilePath
+from pydantic import DirectoryPath, FilePath, RootModel, StringConstraints
 from typing_extensions import Annotated
 
+from .validated_string import ValidatedString
+
 FileName = str
 AbsoluteDirectory = Annotated[DirectoryPath, Predicate(Path.is_absolute)]
 AbsoluteFilePath = Annotated[FilePath, Predicate(Path.is_absolute)]
 
 BIOIMAGEIO_YAML = "bioimageio.yaml"
 ALTERNATIVE_BIOIMAGEIO_YAML_NAMES = ("rdf.yaml", "model.yaml")
 ALL_BIOIMAGEIO_YAML_NAMES = (BIOIMAGEIO_YAML,) + ALTERNATIVE_BIOIMAGEIO_YAML_NAMES
+
+
+class Sha256(ValidatedString):
+    """SHA-256 hash value"""
+
+    root_model: ClassVar[Type[RootModel[Any]]] = RootModel[
+        Annotated[
+            str,
+            StringConstraints(
+                strip_whitespace=True, to_lower=True, min_length=64, max_length=64
+            ),
+        ]
+    ]

bioimageio/spec/_internal/io_utils.py

@@ -32,12 +32,11 @@
     FileDescr,
     HashKwargs,
     OpenedBioimageioYaml,
-    Sha256,
     YamlValue,
     download,
     find_bioimageio_yaml_file_name,
 )
-from .io_basics import FileName
+from .io_basics import FileName, Sha256
 from .types import FileSource, PermissiveFileSource
 
 yaml = YAML(typ="safe")

bioimageio/spec/_internal/types.py

@@ -14,10 +14,10 @@
 from .io import ImportantFileSource as ImportantFileSource
 from .io import PermissiveFileSource as PermissiveFileSource
 from .io import RelativeFilePath as RelativeFilePath
-from .io import Sha256 as Sha256
 from .io_basics import AbsoluteDirectory as AbsoluteDirectory
 from .io_basics import AbsoluteFilePath as AbsoluteFilePath
 from .io_basics import FileName as FileName
+from .io_basics import Sha256 as Sha256
 from .license_id import DeprecatedLicenseId as DeprecatedLicenseId
 from .license_id import LicenseId as LicenseId
 from .url import HttpUrl as HttpUrl

bioimageio/spec/_internal/url.py

@@ -13,7 +13,8 @@
 
 def _validate_url(url: Union[str, pydantic.HttpUrl]) -> pydantic.AnyUrl:
     url = str(url)
-    if not validation_context_var.get().perform_io_checks:
+    context = validation_context_var.get()
+    if not context.perform_io_checks or url in context.known_files:
         return pydantic.AnyUrl(url)
 
     val_url = url

bioimageio/spec/_internal/validation_context.py

@@ -3,13 +3,13 @@
 from contextvars import ContextVar, Token
 from dataclasses import dataclass, field
 from pathlib import Path
-from typing import List, Optional, Union
+from typing import Dict, List, Optional, Union
 from urllib.parse import urlsplit, urlunsplit
 
 from pydantic import DirectoryPath
 
 from ._settings import settings
-from .io_basics import AbsoluteDirectory, FileName
+from .io_basics import AbsoluteDirectory, FileName, Sha256
 from .root_url import RootHttpUrl
 from .warning_levels import WarningLevel
 
@@ -38,13 +38,17 @@ class ValidationContext:
 
     Existence of local absolute file paths is still being checked."""
 
+    known_files: Dict[str, Sha256] = field(default_factory=dict)
+    """allows to bypass download and hashing of referenced files"""
+
     def replace(
         self,
         root: Optional[Union[RootHttpUrl, DirectoryPath]] = None,
         warning_level: Optional[WarningLevel] = None,
         log_warnings: Optional[bool] = None,
         file_name: Optional[str] = None,
         perform_io_checks: Optional[bool] = None,
+        known_files: Optional[Dict[str, Sha256]] = None,
     ) -> "ValidationContext":
         return ValidationContext(
             root=self.root if root is None else root,
@@ -58,6 +62,7 @@ def replace(
                 if perform_io_checks is None
                 else perform_io_checks
             ),
+            known_files=self.known_files if known_files is None else known_files,
         )
 
     def __enter__(self):

bioimageio/spec/application/v0_3.py

@@ -5,8 +5,8 @@
 
 from .._internal.common_nodes import Node
 from .._internal.io import FileDescr as FileDescr
-from .._internal.io import Sha256 as Sha256
 from .._internal.io_basics import AbsoluteFilePath as AbsoluteFilePath
+from .._internal.io_basics import Sha256 as Sha256
 from .._internal.types import ImportantFileSource
 from .._internal.url import HttpUrl as HttpUrl
 from ..generic.v0_3 import VALID_COVER_IMAGE_EXTENSIONS as VALID_COVER_IMAGE_EXTENSIONS

bioimageio/spec/collection/v0_3.py

@@ -19,9 +19,9 @@
 from .._internal.common_nodes import InvalidDescr, Node
 from .._internal.field_warning import issue_warning
 from .._internal.io import FileDescr as FileDescr
-from .._internal.io import Sha256 as Sha256
 from .._internal.io import YamlValue
 from .._internal.io_basics import AbsoluteFilePath as AbsoluteFilePath
+from .._internal.io_basics import Sha256 as Sha256
 from .._internal.io_utils import open_bioimageio_yaml
 from .._internal.types import FileSource, NotEmpty
 from .._internal.url import HttpUrl as HttpUrl

bioimageio/spec/common.py

@@ -4,11 +4,11 @@
 from ._internal.io import BioimageioYamlContent as BioimageioYamlContent
 from ._internal.io import BioimageioYamlSource as BioimageioYamlSource
 from ._internal.io import FileDescr as FileDescr
-from ._internal.io import Sha256 as Sha256
 from ._internal.io import YamlValue as YamlValue
 from ._internal.io_basics import AbsoluteDirectory as AbsoluteDirectory
 from ._internal.io_basics import AbsoluteFilePath as AbsoluteFilePath
 from ._internal.io_basics import FileName as FileName
+from ._internal.io_basics import Sha256 as Sha256
 from ._internal.root_url import RootHttpUrl as RootHttpUrl
 from ._internal.types import FileSource as FileSource
 from ._internal.types import PermissiveFileSource as PermissiveFileSource

bioimageio/spec/dataset/v0_3.py

@@ -4,8 +4,8 @@
 
 from .._internal.common_nodes import InvalidDescr, Node
 from .._internal.io import FileDescr as FileDescr
-from .._internal.io import Sha256 as Sha256
 from .._internal.io_basics import AbsoluteFilePath as AbsoluteFilePath
+from .._internal.io_basics import Sha256 as Sha256
 from .._internal.url import HttpUrl as HttpUrl
 from ..generic.v0_3 import VALID_COVER_IMAGE_EXTENSIONS as VALID_COVER_IMAGE_EXTENSIONS
 from ..generic.v0_3 import Author as Author

bioimageio/spec/generic/v0_3.py

@@ -38,8 +38,8 @@
     validate_suffix,
 )
 from .._internal.io import FileDescr as FileDescr
-from .._internal.io import Sha256 as Sha256
 from .._internal.io_basics import AbsoluteFilePath
+from .._internal.io_basics import Sha256 as Sha256
 from .._internal.license_id import LicenseId
 from .._internal.types import (
     DeprecatedLicenseId,

bioimageio/spec/model/v0_4.py

@@ -47,8 +47,8 @@
     include_in_package_serializer,
 )
 from .._internal.io import FileDescr as FileDescr
-from .._internal.io import Sha256 as Sha256
 from .._internal.io_basics import AbsoluteFilePath as AbsoluteFilePath
+from .._internal.io_basics import Sha256 as Sha256
 from .._internal.packaging_context import packaging_context_var
 from .._internal.types import Datetime as Datetime
 from .._internal.types import Identifier as Identifier

bioimageio/spec/model/v0_5.py

@@ -59,9 +59,9 @@
 from .._internal.field_warning import issue_warning, warn
 from .._internal.io import BioimageioYamlContent as BioimageioYamlContent
 from .._internal.io import FileDescr as FileDescr
-from .._internal.io import Sha256 as Sha256
 from .._internal.io import WithSuffix, YamlValue, download
 from .._internal.io_basics import AbsoluteFilePath as AbsoluteFilePath
+from .._internal.io_basics import Sha256 as Sha256
 from .._internal.io_utils import load_array
 from .._internal.types import Datetime as Datetime
 from .._internal.types import DeprecatedLicenseId as DeprecatedLicenseId

bioimageio/spec/notebook/v0_3.py

@@ -2,8 +2,8 @@
 
 from .._internal.common_nodes import Node
 from .._internal.io import FileDescr as FileDescr
-from .._internal.io import Sha256 as Sha256
 from .._internal.io_basics import AbsoluteFilePath as AbsoluteFilePath
+from .._internal.io_basics import Sha256 as Sha256
 from .._internal.url import HttpUrl as HttpUrl
 from ..generic.v0_3 import VALID_COVER_IMAGE_EXTENSIONS as VALID_COVER_IMAGE_EXTENSIONS
 from ..generic.v0_3 import Author as Author