From f869c2b0a9c681d08cc0c5f533fb59136ad7dbef Mon Sep 17 00:00:00 2001 From: fynnbe Date: Mon, 18 Mar 2024 10:47:19 +0100 Subject: [PATCH 01/36] add default insertion of ensure_dtype procs --- bioimageio/spec/model/v0_5.py | 66 ++++++++++++++++++++++++++++++++--- 1 file changed, 61 insertions(+), 5 deletions(-) diff --git a/bioimageio/spec/model/v0_5.py b/bioimageio/spec/model/v0_5.py index d5ea0fd32..c9292a8c1 100644 --- a/bioimageio/spec/model/v0_5.py +++ b/bioimageio/spec/model/v0_5.py @@ -656,7 +656,19 @@ class ClipDescr(ProcessingDescrBase): class EnsureDtypeKwargs(ProcessingKwargs): - dtype: str + dtype: Literal[ + "float32", + "float64", + "uint8", + "int8", + "uint16", + "int16", + "uint32", + "int32", + "uint64", + "int64", + "bool", + ] class EnsureDtypeDescr(ProcessingDescrBase): @@ -1063,7 +1075,14 @@ class InputTensorDescr(TensorDescrBase[InputAxis]): """indicates that this tensor may be `None`""" preprocessing: List[PreprocessingDescr] = Field(default_factory=list) - """Description of how this input should be preprocessed.""" + """Description of how this input should be preprocessed. + + notes: + - If preprocessing does not start with an 'ensure_dtype' entry, it is added + to ensure a tensor's input data type matches the tensor's data description. + - If preprocessing does not end with an 'ensure_dtype' or 'binarize' entry, it is added + to ensure preprocessing steps are not unintentionally chaning the data type. + """ @model_validator(mode="after") def _validate_preprocessing_kwargs(self) -> Self: @@ -1072,11 +1091,32 @@ def _validate_preprocessing_kwargs(self) -> Self: kwargs_axes: Union[Any, Sequence[Any]] = p.kwargs.get("axes", ()) if not isinstance(kwargs_axes, collections.abc.Sequence): raise ValueError( - f"Expeted `axes` to be a sequence, but got {type(kwargs_axes)}" + f"Expeted `preprocessing.i.kwargs.axes` to be a sequence, but got {type(kwargs_axes)}" ) if any(a not in axes_ids for a in kwargs_axes): - raise ValueError("`kwargs.axes` needs to be subset of axes ids") + raise ValueError( + "`preprocessing.i.kwargs.axes` needs to be subset of axes ids" + ) + + if isinstance(self.data, (NominalOrOrdinalDataDescr, IntervalOrRatioDataDescr)): + dtype = self.data.type + else: + dtype = self.data[0].type + + # ensure `preprocessing` begins with `EnsureDtypeDescr` + if not self.preprocessing or not isinstance( + self.preprocessing[0], EnsureDtypeDescr + ): + self.preprocessing.insert( + 0, EnsureDtypeDescr(kwargs=EnsureDtypeKwargs(dtype=dtype)) + ) + + # ensure `preprocessing` ends with `EnsureDtypeDescr` or `BinarizeDescr` + if not isinstance(self.preprocessing[-1], (EnsureDtypeDescr, BinarizeDescr)): + self.preprocessing.append( + EnsureDtypeDescr(kwargs=EnsureDtypeKwargs(dtype=dtype)) + ) return self @@ -1342,7 +1382,11 @@ class OutputTensorDescr(TensorDescrBase[OutputAxis]): No duplicates are allowed across all inputs and outputs.""" postprocessing: List[PostprocessingDescr] = Field(default_factory=list) - """Description of how this output should be postprocessed.""" + """Description of how this output should be postprocessed. + + note: `postprocessing` always ends with an 'ensure_dtype' operation. + If not given this is added to cast to this tensor's `data.type`. + """ @model_validator(mode="after") def _validate_postprocessing_kwargs(self) -> Self: @@ -1357,6 +1401,18 @@ def _validate_postprocessing_kwargs(self) -> Self: if any(a not in axes_ids for a in kwargs_axes): raise ValueError("`kwargs.axes` needs to be subset of axes ids") + if isinstance(self.data, (NominalOrOrdinalDataDescr, IntervalOrRatioDataDescr)): + dtype = self.data.type + else: + dtype = self.data[0].type + + # ensure `postprocessing` ends with `EnsureDtypeDescr` or `BinarizeDescr` + if not self.postprocessing or not isinstance( + self.postprocessing[-1], (EnsureDtypeDescr, BinarizeDescr) + ): + self.postprocessing.append( + EnsureDtypeDescr(kwargs=EnsureDtypeKwargs(dtype=dtype)) + ) return self From ffdf91d0779299f466a3ef34ef28564c5be0ccd2 Mon Sep 17 00:00:00 2001 From: fynnbe Date: Mon, 18 Mar 2024 10:51:23 +0100 Subject: [PATCH 02/36] bump patch version --- README.md | 7 ++++++- bioimageio/spec/VERSION | 2 +- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 91f84382b..31fdf34a2 100644 --- a/README.md +++ b/README.md @@ -85,6 +85,10 @@ Made with [contrib.rocks](https://contrib.rocks). ### bioimageio.spec Python package +#### bioimageio.spec 0.5.1 + +* new patch version model 0.5.1 + #### bioimageio.spec 0.5.0post2 * don't fail if CI env var is a string @@ -228,10 +232,11 @@ Made with [contrib.rocks](https://contrib.rocks). ### Resource Description Format Versions -#### model 0.5.1 (planned) +#### model 0.5.1 * Non-breaking changes * added optional `inputs.i.optional` field to indicate that a tensor may be `None` + * made data type assumptions in `preprocessing` and `postprocessing` explicit by adding `'ensure_dtype'` operations per default. #### generic 0.3.0 / application 0.3.0 / collection 0.3.0 / dataset 0.3.0 / notebook 0.3.0 diff --git a/bioimageio/spec/VERSION b/bioimageio/spec/VERSION index 0439e3604..4d78bfd6b 100644 --- a/bioimageio/spec/VERSION +++ b/bioimageio/spec/VERSION @@ -1,3 +1,3 @@ { - "version": "0.5.0post2" + "version": "0.5.1" } From 55a0104a666f35f21be4e51f9e1fc25352f2566d Mon Sep 17 00:00:00 2001 From: fynnbe Date: Mon, 18 Mar 2024 14:38:28 +0100 Subject: [PATCH 03/36] remove wasm32 specific code cannot run on pyodide atm --- bioimageio/spec/_internal/io_utils.py | 7 ------- 1 file changed, 7 deletions(-) diff --git a/bioimageio/spec/_internal/io_utils.py b/bioimageio/spec/_internal/io_utils.py index e3fe54b32..51a63e54d 100644 --- a/bioimageio/spec/_internal/io_utils.py +++ b/bioimageio/spec/_internal/io_utils.py @@ -1,5 +1,4 @@ import io -import platform import warnings from contextlib import nullcontext from pathlib import Path @@ -25,12 +24,6 @@ from .io_basics import ALTERNATIVE_BIOIMAGEIO_YAML_NAMES, FileName from .types import FileSource, PermissiveFileSource -if platform.machine() == "wasm32": - import pyodide_http # type: ignore - - pyodide_http.patch_all() - - yaml = YAML(typ="safe") From 3974ba595920fc5302b69337cc363f7cf7e4c706 Mon Sep 17 00:00:00 2001 From: fynnbe Date: Mon, 18 Mar 2024 14:41:31 +0100 Subject: [PATCH 04/36] reformat long line --- bioimageio/spec/summary.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/bioimageio/spec/summary.py b/bioimageio/spec/summary.py index 869db03b9..65f623572 100644 --- a/bioimageio/spec/summary.py +++ b/bioimageio/spec/summary.py @@ -267,7 +267,10 @@ def format_loc(loc: Loc): if d.warnings: details.append(["", "", ""]) - return f"{indent}{self.status_icon} {self.name.strip('.')}: {self.status}\n{src}{env}\n{self._format_md_table(details)}" + return ( + f"{indent}{self.status_icon} {self.name.strip('.')}: {self.status}\n" + + f"{src}{env}\n{self._format_md_table(details)}" + ) @no_type_check def display(self) -> None: From 80a40f3904f5b5b068eaf647093ce73d20eadd25 Mon Sep 17 00:00:00 2001 From: fynnbe Date: Mon, 18 Mar 2024 15:04:32 +0100 Subject: [PATCH 05/36] get_tensor_sizes -> get_axis_sizes --- bioimageio/spec/model/v0_5.py | 22 ++++++---------------- tests/test_model/test_v0_5.py | 8 ++++---- 2 files changed, 10 insertions(+), 20 deletions(-) diff --git a/bioimageio/spec/model/v0_5.py b/bioimageio/spec/model/v0_5.py index f90a68f97..8ce0aa85e 100644 --- a/bioimageio/spec/model/v0_5.py +++ b/bioimageio/spec/model/v0_5.py @@ -1831,13 +1831,6 @@ class _DataDepSize(NamedTuple): max: Optional[int] -class _TensorSizes(NamedTuple): - predetermined: Dict[Tuple[TensorId, AxisId], int] - """size of axis (given `n` for `ParameterizedSize`)""" - data_dependent: Dict[Tuple[TensorId, AxisId], _DataDepSize] - """min,max size of data dependent axis""" - - class ModelDescr(GenericModelDescrBase, title="bioimage.io model specification"): """Specification of the fields used in a bioimage.io-compliant RDF to describe AI models with pretrained weights. These fields are typically stored in a YAML file which we call a model resource description file (model RDF). @@ -2202,15 +2195,14 @@ def get_output_test_arrays(self) -> List[NDArray[Any]]: assert all(isinstance(d, np.ndarray) for d in data) return data - def get_tensor_sizes( + def get_axis_sizes( self, ns: Dict[Tuple[TensorId, AxisId], ParameterizedSize.N], batch_size: int - ) -> _TensorSizes: + ) -> Dict[Tuple[TensorId, AxisId], Union[int, _DataDepSize]]: all_axes = { t.id: {a.id: a for a in t.axes} for t in chain(self.inputs, self.outputs) } - predetermined: Dict[Tuple[TensorId, AxisId], int] = {} - data_dependent: Dict[Tuple[TensorId, AxisId], _DataDepSize] = {} + ret: Dict[Tuple[TensorId, AxisId], Union[int, _DataDepSize]] = {} for t_descr in chain(self.inputs, self.outputs): for a in t_descr.axes: if isinstance(a, BatchAxis): @@ -2247,16 +2239,14 @@ def get_tensor_sizes( raise ValueError( f"No size increment factor (n) for data dependent size axis {a.id} of tensor {t_descr.id} expected." ) - data_dependent[t_descr.id, a.id] = _DataDepSize( - a.size.min, a.size.max - ) + ret[t_descr.id, a.id] = _DataDepSize(a.size.min, a.size.max) continue else: assert_never(a.size) - predetermined[t_descr.id, a.id] = s + ret[t_descr.id, a.id] = s - return _TensorSizes(predetermined, data_dependent) + return ret @model_validator(mode="before") @classmethod diff --git a/tests/test_model/test_v0_5.py b/tests/test_model/test_v0_5.py index 616c6b9bc..dcf38a757 100644 --- a/tests/test_model/test_v0_5.py +++ b/tests/test_model/test_v0_5.py @@ -360,7 +360,7 @@ def test_output_fixed_shape_too_small(model_data: Dict[str, Any]): assert summary.status == "failed", summary.format() -def test_get_tensor_sizes_raises_with_surplus_n(model_data: Dict[str, Any]): +def test_get_axis_sizes_raises_with_surplus_n(model_data: Dict[str, Any]): with ValidationContext(perform_io_checks=False): model = ModelDescr(**model_data) @@ -368,12 +368,12 @@ def test_get_tensor_sizes_raises_with_surplus_n(model_data: Dict[str, Any]): output_axis_id = AxisId("y") with pytest.raises(ValueError): - _ = model.get_tensor_sizes( + _ = model.get_axis_sizes( ns={(output_tensor_id, output_axis_id): 1}, batch_size=1 ) -def test_get_tensor_sizes_raises_with_missing_n(model_data: Dict[str, Any]): +def test_get_axis_sizes_raises_with_missing_n(model_data: Dict[str, Any]): model_data["outputs"][0]["axes"][2] = { "type": "space", "id": "x", @@ -384,7 +384,7 @@ def test_get_tensor_sizes_raises_with_missing_n(model_data: Dict[str, Any]): with ValidationContext(perform_io_checks=False): model = ModelDescr(**model_data) with pytest.raises(ValueError): - _ = model.get_tensor_sizes(ns={}, batch_size=1) + _ = model.get_axis_sizes(ns={}, batch_size=1) def test_output_ref_shape_mismatch(model_data: Dict[str, Any]): From b96190ab9cb087263df469e552197c3414724028 Mon Sep 17 00:00:00 2001 From: fynnbe Date: Mon, 18 Mar 2024 15:45:16 +0100 Subject: [PATCH 06/36] split return of `get_axis_sizes` into input and output tensors --- bioimageio/spec/model/v0_5.py | 23 ++++++++++++++++++----- 1 file changed, 18 insertions(+), 5 deletions(-) diff --git a/bioimageio/spec/model/v0_5.py b/bioimageio/spec/model/v0_5.py index 8ce0aa85e..5dfccca7e 100644 --- a/bioimageio/spec/model/v0_5.py +++ b/bioimageio/spec/model/v0_5.py @@ -1831,6 +1831,11 @@ class _DataDepSize(NamedTuple): max: Optional[int] +class _AxisSizes(NamedTuple): + inputs: Dict[Tuple[TensorId, AxisId], int] + outputs: Dict[Tuple[TensorId, AxisId], Union[int, _DataDepSize]] + + class ModelDescr(GenericModelDescrBase, title="bioimage.io model specification"): """Specification of the fields used in a bioimage.io-compliant RDF to describe AI models with pretrained weights. These fields are typically stored in a YAML file which we call a model resource description file (model RDF). @@ -2197,12 +2202,15 @@ def get_output_test_arrays(self) -> List[NDArray[Any]]: def get_axis_sizes( self, ns: Dict[Tuple[TensorId, AxisId], ParameterizedSize.N], batch_size: int - ) -> Dict[Tuple[TensorId, AxisId], Union[int, _DataDepSize]]: + ) -> _AxisSizes: all_axes = { t.id: {a.id: a for a in t.axes} for t in chain(self.inputs, self.outputs) } - ret: Dict[Tuple[TensorId, AxisId], Union[int, _DataDepSize]] = {} + inputs: Dict[Tuple[TensorId, AxisId], int] = {} + outputs: Dict[Tuple[TensorId, AxisId], Union[int, _DataDepSize]] = {} + input_ids = {d.id for d in self.inputs} + output_ids = {d.id for d in self.outputs} for t_descr in chain(self.inputs, self.outputs): for a in t_descr.axes: if isinstance(a, BatchAxis): @@ -2239,14 +2247,19 @@ def get_axis_sizes( raise ValueError( f"No size increment factor (n) for data dependent size axis {a.id} of tensor {t_descr.id} expected." ) - ret[t_descr.id, a.id] = _DataDepSize(a.size.min, a.size.max) + assert t_descr.id in output_ids + outputs[t_descr.id, a.id] = _DataDepSize(a.size.min, a.size.max) continue else: assert_never(a.size) - ret[t_descr.id, a.id] = s + if t_descr.id in input_ids: + inputs[t_descr.id, a.id] = s + else: + assert t_descr.id in output_ids + outputs[t_descr.id, a.id] = s - return ret + return _AxisSizes(inputs=inputs, outputs=outputs) @model_validator(mode="before") @classmethod From 0c944d98ecf7352568d8649589eb928e15db50f4 Mon Sep 17 00:00:00 2001 From: fynnbe Date: Mon, 18 Mar 2024 20:13:04 +0100 Subject: [PATCH 07/36] add ensure_dtype in example to make roundtrip test pass --- .../models/unet2d_nuclei_broad/bioimageio.yaml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/example_descriptions/models/unet2d_nuclei_broad/bioimageio.yaml b/example_descriptions/models/unet2d_nuclei_broad/bioimageio.yaml index 781358ec6..f7d2ccb8a 100644 --- a/example_descriptions/models/unet2d_nuclei_broad/bioimageio.yaml +++ b/example_descriptions/models/unet2d_nuclei_broad/bioimageio.yaml @@ -52,9 +52,15 @@ inputs: source: test_input.png sha256: 8771f558305dd89f4a85fe659e8ef5e116c94d64e668dc23b9282fda3fe9cce8 preprocessing: # list of preprocessing steps + - id: ensure_dtype # this would be added by default as well + kwargs: + dtype: float32 - id: zero_mean_unit_variance # name of preprocessing step kwargs: axes: [x, y] + - id: ensure_dtype # this would be added by default as well + kwargs: + dtype: float32 outputs: - id: probability From caefb30a684226df3aa8adb3ab87fe8f791db1ad Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fynn=20Beuttenm=C3=BCller?= Date: Tue, 19 Mar 2024 09:01:43 +0100 Subject: [PATCH 08/36] Update bioimageio/spec/model/v0_5.py --- bioimageio/spec/model/v0_5.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/bioimageio/spec/model/v0_5.py b/bioimageio/spec/model/v0_5.py index 5dfccca7e..0d8f4a9ad 100644 --- a/bioimageio/spec/model/v0_5.py +++ b/bioimageio/spec/model/v0_5.py @@ -1119,9 +1119,10 @@ class InputTensorDescr(TensorDescrBase[InputAxis]): notes: - If preprocessing does not start with an 'ensure_dtype' entry, it is added - to ensure a tensor's input data type matches the tensor's data description. - - If preprocessing does not end with an 'ensure_dtype' or 'binarize' entry, it is added - to ensure preprocessing steps are not unintentionally chaning the data type. + to ensure an input tensor's data type matches the input tensor's data description. + - If preprocessing does not end with an 'ensure_dtype' or 'binarize' entry, an + 'ensure_dtype' step is added to ensure preprocessing steps are not unintentionally + changing the data type. """ @model_validator(mode="after") From 557fc25235016aef6ea86c3e3de3a8341fe8e707 Mon Sep 17 00:00:00 2001 From: fynnbe Date: Tue, 19 Mar 2024 10:23:49 +0100 Subject: [PATCH 09/36] add BinarizeAlongAxisKwargs --- bioimageio/spec/model/v0_5.py | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/bioimageio/spec/model/v0_5.py b/bioimageio/spec/model/v0_5.py index c926bda77..287c09523 100644 --- a/bioimageio/spec/model/v0_5.py +++ b/bioimageio/spec/model/v0_5.py @@ -97,7 +97,6 @@ from ..generic.v0_3 import RelativeFilePath as RelativeFilePath from .v0_4 import Author as _Author_v0_4 from .v0_4 import BinarizeDescr as _BinarizeDescr_v0_4 -from .v0_4 import BinarizeKwargs as BinarizeKwargs from .v0_4 import CallableFromDepencency as CallableFromDepencency from .v0_4 import CallableFromDepencency as _CallableFromDepencency_v0_4 from .v0_4 import CallableFromFile as _CallableFromFile_v0_4 @@ -681,13 +680,26 @@ class ProcessingDescrBase(NodeWithExplicitlySetFields, ABC): fields_to_set_explicitly: ClassVar[FrozenSet[LiteralString]] = frozenset({"id"}) +class BinarizeKwargs(ProcessingKwargs): + threshold: float + """The fixed threshold""" + + +class BinarizeAlongAxisKwargs(ProcessingKwargs): + threshold: NotEmpty[List[float]] + """The fixed threshold values along `axis`""" + + axis: Annotated[NonBatchAxisId, Field(examples=["channel"])] + """The `threshold` axis""" + + class BinarizeDescr(ProcessingDescrBase): """Binarize the tensor with a fixed threshold. Values above the threshold will be set to one, values below the threshold to zero. """ id: Literal["binarize"] = "binarize" - kwargs: BinarizeKwargs + kwargs: Union[BinarizeKwargs, BinarizeAlongAxisKwargs] class ClipDescr(ProcessingDescrBase): @@ -1122,8 +1134,8 @@ class InputTensorDescr(TensorDescrBase[InputAxis]): notes: - If preprocessing does not start with an 'ensure_dtype' entry, it is added to ensure an input tensor's data type matches the input tensor's data description. - - If preprocessing does not end with an 'ensure_dtype' or 'binarize' entry, an - 'ensure_dtype' step is added to ensure preprocessing steps are not unintentionally + - If preprocessing does not end with an 'ensure_dtype' or 'binarize' entry, an + 'ensure_dtype' step is added to ensure preprocessing steps are not unintentionally changing the data type. """ From 2b4661450b5381a41cf5b238a2a5e67c10b7072a Mon Sep 17 00:00:00 2001 From: fynnbe Date: Tue, 19 Mar 2024 10:31:47 +0100 Subject: [PATCH 10/36] refactor ScaleLinearKwargs and FixedZeroMeanUnitVarianceKwargs --- bioimageio/spec/model/v0_5.py | 122 +++++++++++++++++++++------------- 1 file changed, 74 insertions(+), 48 deletions(-) diff --git a/bioimageio/spec/model/v0_5.py b/bioimageio/spec/model/v0_5.py index 287c09523..61180c184 100644 --- a/bioimageio/spec/model/v0_5.py +++ b/bioimageio/spec/model/v0_5.py @@ -731,12 +731,26 @@ class EnsureDtypeDescr(ProcessingDescrBase): class ScaleLinearKwargs(ProcessingKwargs): - axis: Annotated[Optional[NonBatchAxisId], Field(examples=["channel"])] = ( - None # todo: validate existence of axis - ) - """The axis of non-scalar gains/offsets. - Invalid for scalar gains/offsets. - """ + gain: float = 1.0 + """multiplicative factor""" + + offset: float = 0.0 + """additive term""" + + @model_validator(mode="after") + def _validate(self) -> Self: + if self.gain == 1.0 and self.offset == 0.0: + raise ValueError( + "Redundant linear scaling not allowd. Set `gain` != 1.0 and/or `offset`" + + " != 0.0." + ) + + return self + + +class ScaleLinearAlongAxisKwargs(ProcessingKwargs): + axis: Annotated[NonBatchAxisId, Field(examples=["channel"])] + """The axis of of gains/offsets values.""" gain: Union[float, NotEmpty[List[float]]] = 1.0 """multiplicative factor""" @@ -745,16 +759,24 @@ class ScaleLinearKwargs(ProcessingKwargs): """additive term""" @model_validator(mode="after") - def either_gain_or_offset(self) -> Self: - if ( - self.gain == 1.0 - or isinstance(self.gain, list) - and all(g == 1.0 for g in self.gain) - ) and ( - self.offset == 0.0 - or isinstance(self.offset, list) - and all(off == 0.0 for off in self.offset) - ): + def _validate(self) -> Self: + + if isinstance(self.gain, list): + if isinstance(self.offset, list): + if len(self.gain) != len(self.offset): + raise ValueError( + f"Size of `gain` ({len(self.gain)}) and `offset` ({len(self.offset)}) must match." + ) + else: + self.offset = [float(self.offset)] * len(self.gain) + elif isinstance(self.offset, list): + self.gain = [float(self.gain)] * len(self.offset) + else: + raise ValueError( + "Do not specify an `axis` for scalar gain and offset values." + ) + + if all(g == 1.0 for g in self.gain) and all(off == 0.0 for off in self.offset): raise ValueError( "Redundant linear scaling not allowd. Set `gain` != 1.0 and/or `offset`" + " != 0.0." @@ -767,7 +789,7 @@ class ScaleLinearDescr(ProcessingDescrBase): """Fixed linear scaling.""" id: Literal["scale_linear"] = "scale_linear" - kwargs: ScaleLinearKwargs + kwargs: Union[ScaleLinearKwargs, ScaleLinearAlongAxisKwargs] class SigmoidDescr(ProcessingDescrBase): @@ -785,34 +807,34 @@ class FixedZeroMeanUnitVarianceKwargs(ProcessingKwargs): """Normalize with fixed, precomputed values for mean and variance. See `zero_mean_unit_variance` for data dependent normalization.""" - mean: Annotated[ - Union[float, NotEmpty[Tuple[float, ...]]], - Field(examples=[3.14, (1.1, -2.2, 3.3)]), - ] - """The mean value(s) to normalize with. Specify `axis` for a sequence of `mean` values""" + mean: float + """The mean value to normalize with.""" - std: Annotated[ - Union[ - Annotated[float, Ge(1e-6)], NotEmpty[Tuple[Annotated[float, Ge(1e-6)], ...]] - ], - Field(examples=[1.05, (0.1, 0.2, 0.3)]), - ] - """The standard deviation value(s) to normalize with. Size must match `mean` values.""" + std: Annotated[float, Ge(1e-6)] + """The standard deviation value to normalize with.""" - axis: Annotated[Optional[NonBatchAxisId], Field(examples=["channel", "index"])] = ( - None # todo: validate existence of axis - ) - """The axis of the mean/std values to normalize each entry along that dimension separately. - Invalid for scalar gains/offsets. - """ + +class FixedZeroMeanUnitVarianceAlongAxisKwargs(ProcessingKwargs): + """Normalize with fixed, precomputed values for mean and variance. + See `zero_mean_unit_variance` for data dependent normalization.""" + + mean: NotEmpty[List[float]] + """The mean value(s) to normalize with.""" + + std: NotEmpty[List[Annotated[float, Ge(1e-6)]]] + """The standard deviation value(s) to normalize with. + Size must match `mean` values.""" + + axis: Annotated[NonBatchAxisId, Field(examples=["channel", "index"])] + """The axis of the mean/std values to normalize each entry along that dimension + separately.""" @model_validator(mode="after") - def mean_and_std_match(self) -> Self: - mean_len = 1 if isinstance(self.mean, (float, int)) else len(self.mean) - std_len = 1 if isinstance(self.std, (float, int)) else len(self.std) - if mean_len != std_len: + def _mean_and_std_match(self) -> Self: + if len(self.mean) != len(self.std): raise ValueError( - "size of `mean` ({mean_len}) and `std` ({std_len}) must match." + f"Size of `mean` ({len(self.mean)}) and `std` ({len(self.std)})" + + " must match." ) return self @@ -822,7 +844,9 @@ class FixedZeroMeanUnitVarianceDescr(ProcessingDescrBase): """Subtract a given mean and divide by a given variance.""" id: Literal["fixed_zero_mean_unit_variance"] = "fixed_zero_mean_unit_variance" - kwargs: FixedZeroMeanUnitVarianceKwargs + kwargs: Union[ + FixedZeroMeanUnitVarianceKwargs, FixedZeroMeanUnitVarianceAlongAxisKwargs + ] class ZeroMeanUnitVarianceKwargs(ProcessingKwargs): @@ -1335,11 +1359,15 @@ def _convert_proc( else: axis = _get_complement_v04_axis(tensor_axes, p.kwargs.axes) - return ScaleLinearDescr( - kwargs=ScaleLinearKwargs( + if axis is None: + assert not isinstance(p.kwargs.gain, list) + assert not isinstance(p.kwargs.offset, list) + kwargs = ScaleLinearKwargs(gain=p.kwargs.gain, offset=p.kwargs.offset) + else: + kwargs = ScaleLinearAlongAxisKwargs( axis=axis, gain=p.kwargs.gain, offset=p.kwargs.offset ) - ) + return ScaleLinearDescr(kwargs=kwargs) elif isinstance(p, _ScaleMeanVarianceDescr_v0_4): return ScaleMeanVarianceDescr( kwargs=ScaleMeanVarianceKwargs( @@ -1352,13 +1380,11 @@ def _convert_proc( if p.kwargs.mode == "fixed": mean = p.kwargs.mean assert mean is not None - if isinstance(mean, list): - mean = tuple(mean) + assert not isinstance(mean, list) std = p.kwargs.std assert std is not None - if isinstance(std, list): - std = tuple(std) + assert not isinstance(std, list) return FixedZeroMeanUnitVarianceDescr( kwargs=FixedZeroMeanUnitVarianceKwargs(mean=mean, std=std) From 1949ba546729b72dc3780da7471411da62596e57 Mon Sep 17 00:00:00 2001 From: fynnbe Date: Tue, 19 Mar 2024 10:34:12 +0100 Subject: [PATCH 11/36] update changelog --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 6367300af..b9a8797da 100644 --- a/README.md +++ b/README.md @@ -238,6 +238,7 @@ Made with [contrib.rocks](https://contrib.rocks). * added `DataDependentSize` for `outputs.i.size` to specify an output shape that is not known before inference is run. * added optional `inputs.i.optional` field to indicate that a tensor may be `None` * made data type assumptions in `preprocessing` and `postprocessing` explicit by adding `'ensure_dtype'` operations per default. + * allow to specify multiple thresholds (along an `axis`) in a 'binarize' processing step #### generic 0.3.0 / application 0.3.0 / collection 0.3.0 / dataset 0.3.0 / notebook 0.3.0 From d1dc684515be29059bfc0e4f43626fc3aca9dbd4 Mon Sep 17 00:00:00 2001 From: fynnbe Date: Tue, 19 Mar 2024 11:00:28 +0100 Subject: [PATCH 12/36] check bioimageio_cache_path env var --- README.md | 4 ++++ bioimageio/spec/__init__.py | 1 + bioimageio/spec/_internal/_settings.py | 5 +++++ bioimageio/spec/_internal/io.py | 1 + tests/test_bioimageio_collection.py | 5 ++++- 5 files changed, 15 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 6367300af..ca02f7422 100644 --- a/README.md +++ b/README.md @@ -71,6 +71,10 @@ or pip install -U bioimageio.core ``` +## 🏞 Environment variables + +TODO: link to settings in dev docs + ## 🤝 How to contribute ## ♥ Contributors diff --git a/bioimageio/spec/__init__.py b/bioimageio/spec/__init__.py index 1ff71e56b..ba476e780 100644 --- a/bioimageio/spec/__init__.py +++ b/bioimageio/spec/__init__.py @@ -13,6 +13,7 @@ from ._description import build_description as build_description from ._description import dump_description as dump_description from ._description import validate_format as validate_format +from ._internal import settings as settings from ._internal.common_nodes import InvalidDescr as InvalidDescr from ._internal.constants import VERSION from ._internal.validation_context import ValidationContext as ValidationContext diff --git a/bioimageio/spec/_internal/_settings.py b/bioimageio/spec/_internal/_settings.py index 78ac500d4..8f80335aa 100644 --- a/bioimageio/spec/_internal/_settings.py +++ b/bioimageio/spec/_internal/_settings.py @@ -1,5 +1,7 @@ +from pathlib import Path from typing import Optional, Union +import pooch from pydantic import Field from pydantic_settings import BaseSettings, SettingsConfigDict from typing_extensions import Annotated @@ -33,6 +35,9 @@ class Settings(BaseSettings, extra="ignore"): user_agent: Optional[str] = None """user agent for http requests""" + cache_path: Path = pooch.os_cache("bioimageio") + """bioimageio cache location""" + @property def github_auth(self): if self.github_username is None or self.github_token is None: diff --git a/bioimageio/spec/_internal/io.py b/bioimageio/spec/_internal/io.py index fc66e9fe0..4ebfec80f 100644 --- a/bioimageio/spec/_internal/io.py +++ b/bioimageio/spec/_internal/io.py @@ -533,6 +533,7 @@ def download( known_hash=_get_known_hash(kwargs), downloader=downloader, fname=fname, + path=settings.cache_path, ) local_source = Path(_ls).absolute() root = strict_source.parent diff --git a/tests/test_bioimageio_collection.py b/tests/test_bioimageio_collection.py index 3baae018a..02364cbae 100644 --- a/tests/test_bioimageio_collection.py +++ b/tests/test_bioimageio_collection.py @@ -6,6 +6,7 @@ import pooch import pytest +from bioimageio.spec import settings from bioimageio.spec._description import DISCOVER, LATEST from bioimageio.spec._internal.types import FormatVersionPlaceholder from tests.utils import ParameterSet, check_bioimageio_yaml @@ -227,7 +228,9 @@ def yield_bioimageio_yaml_urls() -> Iterable[ParameterSet]: - collection_path: Any = pooch.retrieve(BASE_URL + "collection.json", None) + collection_path: Any = pooch.retrieve( + BASE_URL + "collection.json", None, path=settings.cache_path + ) with Path(collection_path).open(encoding="utf-8") as f: collection_data = json.load(f)["collection"] From 7c664864da929862bb7f9b2d9527f983293514c6 Mon Sep 17 00:00:00 2001 From: fynnbe Date: Tue, 19 Mar 2024 11:13:27 +0100 Subject: [PATCH 13/36] restrict valid characters for resource ids --- bioimageio/spec/_internal/types.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/bioimageio/spec/_internal/types.py b/bioimageio/spec/_internal/types.py index c62d04c5e..e81a4528b 100644 --- a/bioimageio/spec/_internal/types.py +++ b/bioimageio/spec/_internal/types.py @@ -1,5 +1,6 @@ from __future__ import annotations +import string from datetime import datetime from keyword import iskeyword from typing import Any, Sequence, TypeVar, Union @@ -22,7 +23,7 @@ from .license_id import LicenseId as LicenseId from .url import HttpUrl as HttpUrl from .validated_string import ValidatedString -from .validator_annotations import AfterValidator, BeforeValidator +from .validator_annotations import AfterValidator, BeforeValidator, RestrictCharacters from .version_type import Version as Version S = TypeVar("S", bound=Sequence[Any]) @@ -117,8 +118,8 @@ class Datetime( _ResourceIdAnno = Annotated[ NotEmpty[str], - annotated_types.LowerCase, - annotated_types.Predicate(lambda s: "\\" not in s and s[0] != "/" and s[-1] != "/"), + RestrictCharacters(string.ascii_lowercase + string.digits + "-/"), + annotated_types.Predicate(lambda s: not (s.startswith("/") or s.endswith("/"))), ] ResourceId = ValidatedString[_ResourceIdAnno] ApplicationId = ValidatedString[_ResourceIdAnno] From b11afb634b54d11cd29521b899d885bd0c2858ed Mon Sep 17 00:00:00 2001 From: fynnbe Date: Tue, 19 Mar 2024 11:22:42 +0100 Subject: [PATCH 14/36] fix conversion of fixed zero mean unit variance --- bioimageio/spec/model/v0_5.py | 29 ++++++++++++++++++++++------- 1 file changed, 22 insertions(+), 7 deletions(-) diff --git a/bioimageio/spec/model/v0_5.py b/bioimageio/spec/model/v0_5.py index 61180c184..c4e53d0e1 100644 --- a/bioimageio/spec/model/v0_5.py +++ b/bioimageio/spec/model/v0_5.py @@ -1379,16 +1379,31 @@ def _convert_proc( elif isinstance(p, _ZeroMeanUnitVarianceDescr_v0_4): if p.kwargs.mode == "fixed": mean = p.kwargs.mean - assert mean is not None - assert not isinstance(mean, list) - std = p.kwargs.std + assert mean is not None assert std is not None - assert not isinstance(std, list) - return FixedZeroMeanUnitVarianceDescr( - kwargs=FixedZeroMeanUnitVarianceKwargs(mean=mean, std=std) - ) + axes = _axes_letters_to_ids(p.kwargs.axes) + axis = _get_complement_v04_axis(tensor_axes, p.kwargs.axes) + + if axis is None: + assert not isinstance(mean, list) + assert not isinstance(std, list) + return FixedZeroMeanUnitVarianceDescr( + kwargs=FixedZeroMeanUnitVarianceKwargs(mean=mean, std=std) + ) + else: + if not isinstance(mean, list): + mean = [float(mean)] + if not isinstance(std, list): + std = [float(std)] + + return FixedZeroMeanUnitVarianceDescr( + kwargs=FixedZeroMeanUnitVarianceAlongAxisKwargs( + axis=axis, mean=mean, std=std + ) + ) + else: axes = _axes_letters_to_ids(p.kwargs.axes) or [] if p.kwargs.mode == "per_dataset": From 12a983e0c22a38129f84034cee620b2686ed77c1 Mon Sep 17 00:00:00 2001 From: fynnbe Date: Tue, 19 Mar 2024 11:39:43 +0100 Subject: [PATCH 15/36] fix _get_complement_v04_axis --- bioimageio/spec/model/v0_5.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/bioimageio/spec/model/v0_5.py b/bioimageio/spec/model/v0_5.py index c4e53d0e1..c46c8a4c2 100644 --- a/bioimageio/spec/model/v0_5.py +++ b/bioimageio/spec/model/v0_5.py @@ -1330,13 +1330,12 @@ def _get_complement_v04_axis( if axes is None: return None - axes_str = str(axes) - all_axes = set(str(tensor_axes)) | {"b"} - complement_axes = [a for a in axes_str if a not in all_axes] + non_complement_axes = set(axes) | {"b"} + complement_axes = [a for a in tensor_axes if a not in non_complement_axes] if len(complement_axes) > 1: raise ValueError( f"Expected none or a single complement axis, but axes '{axes}' " - + f"for tensor dims '{all_axes}' leave '{complement_axes}'." + + f"for tensor dims '{tensor_axes}' leave '{complement_axes}'." ) return None if not complement_axes else AxisId(complement_axes[0]) @@ -1383,7 +1382,6 @@ def _convert_proc( assert mean is not None assert std is not None - axes = _axes_letters_to_ids(p.kwargs.axes) axis = _get_complement_v04_axis(tensor_axes, p.kwargs.axes) if axis is None: From 1f210b0839b3670ec343ff0a00f994f6cc409d45 Mon Sep 17 00:00:00 2001 From: fynnbe Date: Tue, 19 Mar 2024 11:56:36 +0100 Subject: [PATCH 16/36] add docstring --- bioimageio/spec/_internal/io.py | 1 + 1 file changed, 1 insertion(+) diff --git a/bioimageio/spec/_internal/io.py b/bioimageio/spec/_internal/io.py index fc66e9fe0..4e52da679 100644 --- a/bioimageio/spec/_internal/io.py +++ b/bioimageio/spec/_internal/io.py @@ -502,6 +502,7 @@ def download( /, **kwargs: Unpack[HashKwargs], ) -> DownloadedFile: + """download `source` URL (or pass local file path)""" if isinstance(source, FileDescr): return source.download() From 8c9b7fe8d8a3da6d7df22150f8edbaaa7a4ae8e9 Mon Sep 17 00:00:00 2001 From: fynnbe Date: Tue, 19 Mar 2024 12:02:38 +0100 Subject: [PATCH 17/36] mark 10.5281/zenodo.7274275/8123818/rdf.yaml as invalid as latest --- bioimageio/spec/model/v0_5.py | 6 +++--- tests/test_bioimageio_collection.py | 1 + 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/bioimageio/spec/model/v0_5.py b/bioimageio/spec/model/v0_5.py index c46c8a4c2..670ab39d3 100644 --- a/bioimageio/spec/model/v0_5.py +++ b/bioimageio/spec/model/v0_5.py @@ -1385,10 +1385,10 @@ def _convert_proc( axis = _get_complement_v04_axis(tensor_axes, p.kwargs.axes) if axis is None: - assert not isinstance(mean, list) - assert not isinstance(std, list) return FixedZeroMeanUnitVarianceDescr( - kwargs=FixedZeroMeanUnitVarianceKwargs(mean=mean, std=std) + kwargs=FixedZeroMeanUnitVarianceKwargs( + mean=mean, std=std # pyright: ignore[reportArgumentType] + ) ) else: if not isinstance(mean, list): diff --git a/tests/test_bioimageio_collection.py b/tests/test_bioimageio_collection.py index 3baae018a..c375ddca2 100644 --- a/tests/test_bioimageio_collection.py +++ b/tests/test_bioimageio_collection.py @@ -73,6 +73,7 @@ "10.5281/zenodo.6559929/6559930/rdf.yaml", "10.5281/zenodo.6811491/6811492/rdf.yaml", "10.5281/zenodo.6865412/6919253/rdf.yaml", + "10.5281/zenodo.7274275/8123818/rdf.yaml", "10.5281/zenodo.7380171/7405349/rdf.yaml", "10.5281/zenodo.7614645/7642674/rdf.yaml", "10.5281/zenodo.8401064/8429203/rdf.yaml", From 4c558fb16a7a885590fc49603f1f4e4422640068 Mon Sep 17 00:00:00 2001 From: fynnbe Date: Tue, 19 Mar 2024 13:46:06 +0100 Subject: [PATCH 18/36] move resource id types to their respective submodules --- bioimageio/spec/_internal/types.py | 15 +-------------- .../spec/_internal/validator_annotations.py | 6 +++++- bioimageio/spec/application/v0_2.py | 6 ++++-- bioimageio/spec/application/v0_3.py | 6 ++++-- bioimageio/spec/collection/v0_2.py | 15 +++++---------- bioimageio/spec/collection/v0_3.py | 14 +++++--------- bioimageio/spec/dataset/v0_2.py | 6 ++++-- bioimageio/spec/dataset/v0_3.py | 5 ++++- bioimageio/spec/generic/v0_2.py | 14 ++++++++++++-- bioimageio/spec/generic/v0_3.py | 11 ++++++++++- bioimageio/spec/model/v0_4.py | 7 ++++--- bioimageio/spec/model/v0_5.py | 10 ++++++---- bioimageio/spec/notebook/v0_2.py | 7 ++++--- bioimageio/spec/notebook/v0_3.py | 6 ++++-- tests/test_internal/test_types.py | 6 ------ 15 files changed, 72 insertions(+), 62 deletions(-) diff --git a/bioimageio/spec/_internal/types.py b/bioimageio/spec/_internal/types.py index e81a4528b..07dd8aadf 100644 --- a/bioimageio/spec/_internal/types.py +++ b/bioimageio/spec/_internal/types.py @@ -1,6 +1,5 @@ from __future__ import annotations -import string from datetime import datetime from keyword import iskeyword from typing import Any, Sequence, TypeVar, Union @@ -23,7 +22,7 @@ from .license_id import LicenseId as LicenseId from .url import HttpUrl as HttpUrl from .validated_string import ValidatedString -from .validator_annotations import AfterValidator, BeforeValidator, RestrictCharacters +from .validator_annotations import AfterValidator, BeforeValidator from .version_type import Version as Version S = TypeVar("S", bound=Sequence[Any]) @@ -115,15 +114,3 @@ class Datetime( ), ] ] - -_ResourceIdAnno = Annotated[ - NotEmpty[str], - RestrictCharacters(string.ascii_lowercase + string.digits + "-/"), - annotated_types.Predicate(lambda s: not (s.startswith("/") or s.endswith("/"))), -] -ResourceId = ValidatedString[_ResourceIdAnno] -ApplicationId = ValidatedString[_ResourceIdAnno] -CollectionId = ValidatedString[_ResourceIdAnno] -DatasetId = ValidatedString[_ResourceIdAnno] -ModelId = ValidatedString[_ResourceIdAnno] -NotebookId = ValidatedString[_ResourceIdAnno] diff --git a/bioimageio/spec/_internal/validator_annotations.py b/bioimageio/spec/_internal/validator_annotations.py index 5dce96e2a..00e396524 100644 --- a/bioimageio/spec/_internal/validator_annotations.py +++ b/bioimageio/spec/_internal/validator_annotations.py @@ -45,9 +45,13 @@ def __get_pydantic_core_schema__( ) -> CoreSchema: if not self.alphabet: raise ValueError("Alphabet may not be empty") + schema = handler(source) # get the CoreSchema from the type / inner constraints - if schema["type"] != "str": + if schema["type"] != "str" and not ( + schema["type"] == "function-after" and schema["schema"]["type"] == "str" + ): raise TypeError("RestrictCharacters can only be applied to strings") + return no_info_after_validator_function( self.validate, schema, diff --git a/bioimageio/spec/application/v0_2.py b/bioimageio/spec/application/v0_2.py index 2a4b67afd..3e799437d 100644 --- a/bioimageio/spec/application/v0_2.py +++ b/bioimageio/spec/application/v0_2.py @@ -5,15 +5,15 @@ from .._internal.common_nodes import Node from .._internal.io_basics import AbsoluteFilePath as AbsoluteFilePath -from .._internal.types import ApplicationId as ApplicationId from .._internal.types import ImportantFileSource from .._internal.url import HttpUrl as HttpUrl +from .._internal.validated_string import ValidatedString from ..generic.v0_2 import AttachmentsDescr as AttachmentsDescr from ..generic.v0_2 import Author as Author from ..generic.v0_2 import BadgeDescr as BadgeDescr from ..generic.v0_2 import CiteEntry as CiteEntry from ..generic.v0_2 import Doi as Doi -from ..generic.v0_2 import GenericDescrBase +from ..generic.v0_2 import GenericDescrBase, ResourceId_v0_2_Anno from ..generic.v0_2 import LinkedResource as LinkedResource from ..generic.v0_2 import Maintainer as Maintainer from ..generic.v0_2 import OrcidId as OrcidId @@ -22,6 +22,8 @@ from ..generic.v0_2 import Uploader as Uploader from ..generic.v0_2 import Version as Version +ApplicationId = ValidatedString[ResourceId_v0_2_Anno] + class ApplicationDescr(GenericDescrBase, title="bioimage.io application specification"): """Bioimage.io description of an application.""" diff --git a/bioimageio/spec/application/v0_3.py b/bioimageio/spec/application/v0_3.py index 31ae5eb51..acc27830a 100644 --- a/bioimageio/spec/application/v0_3.py +++ b/bioimageio/spec/application/v0_3.py @@ -7,14 +7,14 @@ from .._internal.io import FileDescr as FileDescr from .._internal.io import Sha256 as Sha256 from .._internal.io_basics import AbsoluteFilePath as AbsoluteFilePath -from .._internal.types import ApplicationId as ApplicationId from .._internal.types import ImportantFileSource from .._internal.url import HttpUrl as HttpUrl +from .._internal.validated_string import ValidatedString from ..generic.v0_3 import Author as Author from ..generic.v0_3 import BadgeDescr as BadgeDescr from ..generic.v0_3 import CiteEntry as CiteEntry from ..generic.v0_3 import Doi as Doi -from ..generic.v0_3 import GenericDescrBase +from ..generic.v0_3 import GenericDescrBase, ResourceIdAnno from ..generic.v0_3 import LinkedResource as LinkedResource from ..generic.v0_3 import Maintainer as Maintainer from ..generic.v0_3 import OrcidId as OrcidId @@ -23,6 +23,8 @@ from ..generic.v0_3 import Uploader as Uploader from ..generic.v0_3 import Version as Version +ApplicationId = ValidatedString[ResourceIdAnno] + class ApplicationDescr(GenericDescrBase, title="bioimage.io application specification"): """Bioimage.io description of an application.""" diff --git a/bioimageio/spec/collection/v0_2.py b/bioimageio/spec/collection/v0_2.py index 7994eb14f..763e699c3 100644 --- a/bioimageio/spec/collection/v0_2.py +++ b/bioimageio/spec/collection/v0_2.py @@ -15,13 +15,9 @@ from .._internal.io import BioimageioYamlContent, YamlValue from .._internal.io_basics import AbsoluteFilePath as AbsoluteFilePath from .._internal.io_utils import open_bioimageio_yaml -from .._internal.types import ApplicationId as ApplicationId -from .._internal.types import CollectionId as CollectionId -from .._internal.types import DatasetId as DatasetId -from .._internal.types import ModelId as ModelId -from .._internal.types import NotebookId as NotebookId from .._internal.types import NotEmpty from .._internal.url import HttpUrl as HttpUrl +from .._internal.validated_string import ValidatedString from .._internal.validation_context import validation_context_var from .._internal.warning_levels import ALERT from ..application import ApplicationDescr_v0_2, ApplicationDescr_v0_3 @@ -32,17 +28,18 @@ from ..generic.v0_2 import BadgeDescr as BadgeDescr from ..generic.v0_2 import CiteEntry as CiteEntry from ..generic.v0_2 import Doi as Doi -from ..generic.v0_2 import FileSource, GenericDescrBase +from ..generic.v0_2 import FileSource, GenericDescrBase, ResourceId_v0_2_Anno from ..generic.v0_2 import LinkedResource as LinkedResource from ..generic.v0_2 import Maintainer as Maintainer from ..generic.v0_2 import OrcidId as OrcidId from ..generic.v0_2 import RelativeFilePath as RelativeFilePath -from ..generic.v0_2 import ResourceId as ResourceId from ..generic.v0_2 import Uploader as Uploader from ..generic.v0_2 import Version as Version from ..model import ModelDescr_v0_4, ModelDescr_v0_5 from ..notebook import NotebookDescr_v0_2, NotebookDescr_v0_3 +CollectionId = ValidatedString[ResourceId_v0_2_Anno] + EntryDescr = Union[ ApplicationDescr_v0_2, DatasetDescr_v0_2, @@ -113,9 +110,7 @@ class CollectionEntry(Node, extra="allow"): rdf_source: Optional[FileSource] = None """resource description file (RDF) source to load entry from""" - id: Optional[Union[ResourceId, DatasetId, ApplicationId, ModelId, NotebookId]] = ( - None - ) + id: Optional[ResourceId_v0_2_Anno] = None """Collection entry sub id overwriting `rdf_source.id`. The full collection entry's id is the collection's base id, followed by this sub id and separated by a slash '/'.""" diff --git a/bioimageio/spec/collection/v0_3.py b/bioimageio/spec/collection/v0_3.py index d21fbdc58..9b77aecba 100644 --- a/bioimageio/spec/collection/v0_3.py +++ b/bioimageio/spec/collection/v0_3.py @@ -23,13 +23,9 @@ from .._internal.io import YamlValue from .._internal.io_basics import AbsoluteFilePath as AbsoluteFilePath from .._internal.io_utils import open_bioimageio_yaml -from .._internal.types import ApplicationId as ApplicationId -from .._internal.types import CollectionId as CollectionId -from .._internal.types import DatasetId as DatasetId from .._internal.types import FileSource, NotEmpty -from .._internal.types import ModelId as ModelId -from .._internal.types import NotebookId as NotebookId from .._internal.url import HttpUrl as HttpUrl +from .._internal.validated_string import ValidatedString from .._internal.validation_context import ( validation_context_var, ) @@ -43,6 +39,7 @@ from ..generic.v0_3 import Doi as Doi from ..generic.v0_3 import ( GenericDescrBase, + ResourceIdAnno, _author_conv, # pyright: ignore[reportPrivateUsage] _maintainer_conv, # pyright: ignore[reportPrivateUsage] ) @@ -50,13 +47,14 @@ from ..generic.v0_3 import Maintainer as Maintainer from ..generic.v0_3 import OrcidId as OrcidId from ..generic.v0_3 import RelativeFilePath as RelativeFilePath -from ..generic.v0_3 import ResourceId as ResourceId from ..generic.v0_3 import Uploader as Uploader from ..generic.v0_3 import Version as Version from ..model import ModelDescr_v0_4, ModelDescr_v0_5 from ..notebook import NotebookDescr_v0_2, NotebookDescr_v0_3 from .v0_2 import CollectionDescr as _CollectionDescr_v0_2 +CollectionId = ValidatedString[ResourceIdAnno] + EntryDescr = Union[ ApplicationDescr_v0_2, ApplicationDescr_v0_3, @@ -131,9 +129,7 @@ class CollectionEntry(Node, extra="allow"): entry_source: Optional[FileSource] = None """an external source this entry description is based on""" - id: Optional[Union[ResourceId, DatasetId, ApplicationId, ModelId, NotebookId]] = ( - None - ) + id: Optional[ResourceIdAnno] = None """Collection entry sub id overwriting `rdf_source.id`. The full collection entry's id is the collection's base id, followed by this sub id and separated by a slash '/'.""" diff --git a/bioimageio/spec/dataset/v0_2.py b/bioimageio/spec/dataset/v0_2.py index 4c4892846..4adb34535 100644 --- a/bioimageio/spec/dataset/v0_2.py +++ b/bioimageio/spec/dataset/v0_2.py @@ -2,14 +2,14 @@ from .._internal.common_nodes import Node from .._internal.io_basics import AbsoluteFilePath as AbsoluteFilePath -from .._internal.types import DatasetId as DatasetId from .._internal.url import HttpUrl as HttpUrl +from .._internal.validated_string import ValidatedString from ..generic.v0_2 import AttachmentsDescr as AttachmentsDescr from ..generic.v0_2 import Author as Author from ..generic.v0_2 import BadgeDescr as BadgeDescr from ..generic.v0_2 import CiteEntry as CiteEntry from ..generic.v0_2 import Doi as Doi -from ..generic.v0_2 import GenericDescrBase +from ..generic.v0_2 import GenericDescrBase, ResourceId_v0_2_Anno from ..generic.v0_2 import LinkedResource as LinkedResource from ..generic.v0_2 import Maintainer as Maintainer from ..generic.v0_2 import OrcidId as OrcidId @@ -18,6 +18,8 @@ from ..generic.v0_2 import Uploader as Uploader from ..generic.v0_2 import Version as Version +DatasetId = ValidatedString[ResourceId_v0_2_Anno] + class DatasetDescr(GenericDescrBase, title="bioimage.io dataset specification"): """A bioimage.io dataset resource description file (dataset RDF) describes a dataset relevant to bioimage diff --git a/bioimageio/spec/dataset/v0_3.py b/bioimageio/spec/dataset/v0_3.py index b50738622..79922eaec 100644 --- a/bioimageio/spec/dataset/v0_3.py +++ b/bioimageio/spec/dataset/v0_3.py @@ -6,14 +6,15 @@ from .._internal.io import FileDescr as FileDescr from .._internal.io import Sha256 as Sha256 from .._internal.io_basics import AbsoluteFilePath as AbsoluteFilePath -from .._internal.types import DatasetId as DatasetId from .._internal.url import HttpUrl as HttpUrl +from .._internal.validated_string import ValidatedString from ..generic.v0_3 import Author as Author from ..generic.v0_3 import BadgeDescr as BadgeDescr from ..generic.v0_3 import CiteEntry as CiteEntry from ..generic.v0_3 import ( DocumentationSource, GenericDescrBase, + ResourceIdAnno, _author_conv, # pyright: ignore[reportPrivateUsage] _maintainer_conv, # pyright: ignore[reportPrivateUsage] ) @@ -27,6 +28,8 @@ from ..generic.v0_3 import Version as Version from .v0_2 import DatasetDescr as DatasetDescr02 +DatasetId = ValidatedString[ResourceIdAnno] + class DatasetDescr(GenericDescrBase, title="bioimage.io dataset specification"): """A bioimage.io dataset resource description file (dataset RDF) describes a dataset relevant to bioimage diff --git a/bioimageio/spec/generic/v0_2.py b/bioimageio/spec/generic/v0_2.py index 9765c6a76..617dc0625 100644 --- a/bioimageio/spec/generic/v0_2.py +++ b/bioimageio/spec/generic/v0_2.py @@ -1,4 +1,5 @@ import collections.abc +import string from typing import ( Any, Dict, @@ -11,6 +12,7 @@ Union, ) +import annotated_types from annotated_types import Len, LowerCase, MaxLen from pydantic import EmailStr, Field, ValidationInfo, field_validator, model_validator from typing_extensions import Annotated, Self, assert_never @@ -35,12 +37,20 @@ from .._internal.types import Doi as Doi from .._internal.types import OrcidId as OrcidId from .._internal.types import RelativeFilePath as RelativeFilePath -from .._internal.types import ResourceId as ResourceId from .._internal.url import HttpUrl as HttpUrl -from .._internal.validator_annotations import AfterValidator +from .._internal.validated_string import ValidatedString +from .._internal.validator_annotations import AfterValidator, RestrictCharacters from .._internal.version_type import Version as Version from ._v0_2_converter import convert_from_older_format as _convert_from_older_format +ResourceId_v0_2_Anno = Annotated[ + NotEmpty[str], + AfterValidator(lambda s: s.lower()), # convert upper case on the fly + RestrictCharacters(string.ascii_lowercase + string.digits + "_-/"), + annotated_types.Predicate(lambda s: not (s.startswith("/") or s.endswith("/"))), +] +ResourceId = ValidatedString[ResourceId_v0_2_Anno] + KNOWN_SPECIFIC_RESOURCE_TYPES = ( "application", "collection", diff --git a/bioimageio/spec/generic/v0_3.py b/bioimageio/spec/generic/v0_3.py index 2ecc4b578..8eb75bace 100644 --- a/bioimageio/spec/generic/v0_3.py +++ b/bioimageio/spec/generic/v0_3.py @@ -4,11 +4,13 @@ from functools import partial from typing import Any, Dict, List, Literal, Optional, Sequence, TypeVar, Union +import annotated_types from annotated_types import Len, LowerCase, MaxLen, MinLen from pydantic import Field, ValidationInfo, field_validator, model_validator from typing_extensions import Annotated from bioimageio.spec._internal.field_validation import validate_gh_user +from bioimageio.spec._internal.validated_string import ValidatedString from .._internal.common_nodes import ( Converter, @@ -36,7 +38,6 @@ NotEmpty, ) from .._internal.types import RelativeFilePath as RelativeFilePath -from .._internal.types import ResourceId as ResourceId from .._internal.url import HttpUrl as HttpUrl from .._internal.validator_annotations import ( AfterValidator, @@ -62,6 +63,14 @@ "notebook", ) +ResourceIdAnno = Annotated[ + NotEmpty[str], + RestrictCharacters(string.ascii_lowercase + string.digits + "_-/"), + annotated_types.Predicate(lambda s: not (s.startswith("/") or s.endswith("/"))), +] + +ResourceId = ValidatedString[ResourceIdAnno] + def _validate_md_suffix(value: V_suffix) -> V_suffix: return validate_suffix(value, suffix=".md", case_sensitive=True) diff --git a/bioimageio/spec/model/v0_4.py b/bioimageio/spec/model/v0_4.py index efd83f1ef..92941fc2c 100644 --- a/bioimageio/spec/model/v0_4.py +++ b/bioimageio/spec/model/v0_4.py @@ -54,9 +54,7 @@ LowerCaseIdentifierAnno, ) from .._internal.types import LicenseId as LicenseId -from .._internal.types import ModelId as ModelId from .._internal.types import NotEmpty as NotEmpty -from .._internal.types import ResourceId as ResourceId from .._internal.url import HttpUrl as HttpUrl from .._internal.validator_annotations import AfterValidator, RestrictCharacters from .._internal.version_type import Version as Version @@ -68,15 +66,18 @@ from ..generic.v0_2 import BadgeDescr as BadgeDescr from ..generic.v0_2 import CiteEntry as CiteEntry from ..generic.v0_2 import Doi as Doi -from ..generic.v0_2 import GenericModelDescrBase +from ..generic.v0_2 import GenericModelDescrBase, ResourceId_v0_2_Anno from ..generic.v0_2 import LinkedResource as LinkedResource from ..generic.v0_2 import Maintainer as Maintainer from ..generic.v0_2 import OrcidId as OrcidId from ..generic.v0_2 import RelativeFilePath as RelativeFilePath from ..generic.v0_2 import Uploader as Uploader +from ..generic.v0_3 import ResourceId as ResourceId from ..utils import load_array from ._v0_4_converter import convert_from_older_format +ModelId = ValidatedString[ResourceId_v0_2_Anno] + AxesStr = Annotated[ str, RestrictCharacters("bitczyx"), AfterValidator(validate_unique_entries) ] diff --git a/bioimageio/spec/model/v0_5.py b/bioimageio/spec/model/v0_5.py index c926bda77..bf7788416 100644 --- a/bioimageio/spec/model/v0_5.py +++ b/bioimageio/spec/model/v0_5.py @@ -51,6 +51,7 @@ from bioimageio.spec._internal.validated_string import ValidatedString from bioimageio.spec._internal.validator_annotations import RestrictCharacters +from bioimageio.spec.generic.v0_3 import ResourceIdAnno from .._internal.common_nodes import ( Converter, @@ -71,9 +72,7 @@ from .._internal.types import Identifier as Identifier from .._internal.types import ImportantFileSource, LowerCaseIdentifierAnno, SiUnit from .._internal.types import LicenseId as LicenseId -from .._internal.types import ModelId as ModelId from .._internal.types import NotEmpty as NotEmpty -from .._internal.types import ResourceId as ResourceId from .._internal.url import HttpUrl as HttpUrl from .._internal.validation_context import validation_context_var from .._internal.version_type import Version as Version @@ -1122,8 +1121,8 @@ class InputTensorDescr(TensorDescrBase[InputAxis]): notes: - If preprocessing does not start with an 'ensure_dtype' entry, it is added to ensure an input tensor's data type matches the input tensor's data description. - - If preprocessing does not end with an 'ensure_dtype' or 'binarize' entry, an - 'ensure_dtype' step is added to ensure preprocessing steps are not unintentionally + - If preprocessing does not end with an 'ensure_dtype' or 'binarize' entry, an + 'ensure_dtype' step is added to ensure preprocessing steps are not unintentionally changing the data type. """ @@ -1819,6 +1818,9 @@ def check_entries(self) -> Self: return self +ModelId = ValidatedString[ResourceIdAnno] + + class LinkedModel(Node): """Reference to a bioimage.io model.""" diff --git a/bioimageio/spec/notebook/v0_2.py b/bioimageio/spec/notebook/v0_2.py index 13bb578fa..21b564188 100644 --- a/bioimageio/spec/notebook/v0_2.py +++ b/bioimageio/spec/notebook/v0_2.py @@ -5,22 +5,23 @@ from .._internal.common_nodes import Node from .._internal.io import WithSuffix from .._internal.io_basics import AbsoluteFilePath as AbsoluteFilePath -from .._internal.types import NotebookId as NotebookId from .._internal.url import HttpUrl +from .._internal.validated_string import ValidatedString from ..generic.v0_2 import AttachmentsDescr as AttachmentsDescr from ..generic.v0_2 import Author as Author from ..generic.v0_2 import BadgeDescr as BadgeDescr from ..generic.v0_2 import CiteEntry as CiteEntry from ..generic.v0_2 import Doi as Doi -from ..generic.v0_2 import GenericDescrBase +from ..generic.v0_2 import GenericDescrBase, ResourceId_v0_2_Anno from ..generic.v0_2 import LinkedResource as LinkedResource from ..generic.v0_2 import Maintainer as Maintainer from ..generic.v0_2 import OrcidId as OrcidId from ..generic.v0_2 import RelativeFilePath as RelativeFilePath -from ..generic.v0_2 import ResourceId as ResourceId from ..generic.v0_2 import Uploader as Uploader from ..generic.v0_2 import Version as Version +NotebookId = ValidatedString[ResourceId_v0_2_Anno] + _WithNotebookSuffix = WithSuffix(".ipynb", case_sensitive=True) NotebookSource = Union[ Annotated[HttpUrl, _WithNotebookSuffix], diff --git a/bioimageio/spec/notebook/v0_3.py b/bioimageio/spec/notebook/v0_3.py index 214c30eb6..86114a086 100644 --- a/bioimageio/spec/notebook/v0_3.py +++ b/bioimageio/spec/notebook/v0_3.py @@ -4,13 +4,13 @@ from .._internal.io import FileDescr as FileDescr from .._internal.io import Sha256 as Sha256 from .._internal.io_basics import AbsoluteFilePath as AbsoluteFilePath -from .._internal.types import NotebookId as NotebookId from .._internal.url import HttpUrl as HttpUrl +from .._internal.validated_string import ValidatedString from ..generic.v0_3 import Author as Author from ..generic.v0_3 import BadgeDescr as BadgeDescr from ..generic.v0_3 import CiteEntry as CiteEntry from ..generic.v0_3 import Doi as Doi -from ..generic.v0_3 import GenericDescrBase +from ..generic.v0_3 import GenericDescrBase, ResourceIdAnno from ..generic.v0_3 import LinkedResource as LinkedResource from ..generic.v0_3 import Maintainer as Maintainer from ..generic.v0_3 import OrcidId as OrcidId @@ -20,6 +20,8 @@ from ..generic.v0_3 import Version as Version from .v0_2 import NotebookSource as NotebookSource +NotebookId = ValidatedString[ResourceIdAnno] + class NotebookDescr(GenericDescrBase, title="bioimage.io notebook specification"): """Bioimage.io description of a Jupyter notebook.""" diff --git a/tests/test_internal/test_types.py b/tests/test_internal/test_types.py index e5aebe962..b886d560b 100644 --- a/tests/test_internal/test_types.py +++ b/tests/test_internal/test_types.py @@ -13,9 +13,6 @@ from tests.utils import check_type TYPE_ARGS = { - types.ApplicationId: "appdev/app", - types.CollectionId: "collectionid", - types.DatasetId: "dataset-id", types.Datetime: (2024, 2, 14), types.Datetime: datetime.now().isoformat(), types.DeprecatedLicenseId: "AGPL-1.0", @@ -26,11 +23,8 @@ types.LicenseId: "MIT", types.LowerCaseIdentifier: "id", types.LowerCaseIdentifierAnno: "id", - types.ModelId: "modelid", - types.NotebookId: "notebookid", types.OrcidId: "0000-0001-2345-6789", types.RelativeFilePath: Path(__file__).relative_to(Path().absolute()), - types.ResourceId: "resoruce-id", types.SiUnit: "kg", types.AbsoluteDirectory: str(Path(__file__).absolute().parent), types.AbsoluteFilePath: str(Path(__file__).absolute()), From f699cb9b1aa71c91ff3c45e2bd4e1c0cadf924ad Mon Sep 17 00:00:00 2001 From: fynnbe Date: Tue, 19 Mar 2024 13:55:45 +0100 Subject: [PATCH 19/36] fix test_specific_reexports_generics --- tests/test_specific_reexports_generics.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/test_specific_reexports_generics.py b/tests/test_specific_reexports_generics.py index 1a27ff103..4baa2e152 100644 --- a/tests/test_specific_reexports_generics.py +++ b/tests/test_specific_reexports_generics.py @@ -8,6 +8,7 @@ IGNORE_MEMBERS = { "AfterValidator", "ALERT", + "annotated_types", "Annotated", "annotations", "Any", @@ -47,8 +48,6 @@ "partial", "Predicate", "requests", - "ResourceDescrBase", - "ResourceDescrType", "RestrictCharacters", "Self", "Sequence", @@ -85,6 +84,9 @@ def get_members(m: ModuleType): "GenericDescrBase", "GenericModelDescrBase", "KNOWN_SPECIFIC_RESOURCE_TYPES", + "ResourceDescrBase", + "ResourceDescrType", + "ResourceId", "VALID_COVER_IMAGE_EXTENSIONS", } From d4ec59110fed02fe9c9a7fa8e385e27c2d982516 Mon Sep 17 00:00:00 2001 From: fynnbe Date: Tue, 19 Mar 2024 13:57:50 +0100 Subject: [PATCH 20/36] allow dots in ids --- bioimageio/spec/generic/v0_2.py | 2 +- bioimageio/spec/generic/v0_3.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/bioimageio/spec/generic/v0_2.py b/bioimageio/spec/generic/v0_2.py index 617dc0625..36a3c75cb 100644 --- a/bioimageio/spec/generic/v0_2.py +++ b/bioimageio/spec/generic/v0_2.py @@ -46,7 +46,7 @@ ResourceId_v0_2_Anno = Annotated[ NotEmpty[str], AfterValidator(lambda s: s.lower()), # convert upper case on the fly - RestrictCharacters(string.ascii_lowercase + string.digits + "_-/"), + RestrictCharacters(string.ascii_lowercase + string.digits + "_-/."), annotated_types.Predicate(lambda s: not (s.startswith("/") or s.endswith("/"))), ] ResourceId = ValidatedString[ResourceId_v0_2_Anno] diff --git a/bioimageio/spec/generic/v0_3.py b/bioimageio/spec/generic/v0_3.py index 8eb75bace..0a648ebdc 100644 --- a/bioimageio/spec/generic/v0_3.py +++ b/bioimageio/spec/generic/v0_3.py @@ -65,7 +65,7 @@ ResourceIdAnno = Annotated[ NotEmpty[str], - RestrictCharacters(string.ascii_lowercase + string.digits + "_-/"), + RestrictCharacters(string.ascii_lowercase + string.digits + "_-/."), annotated_types.Predicate(lambda s: not (s.startswith("/") or s.endswith("/"))), ] From 42731805618689482b52df4201a399c2d418b926 Mon Sep 17 00:00:00 2001 From: fynnbe Date: Tue, 19 Mar 2024 14:03:08 +0100 Subject: [PATCH 21/36] update git_repo urls in tests --- tests/test_model/test_v0_4.py | 2 +- tests/test_model/test_v0_5.py | 30 ++++++++++++++++++++++-------- 2 files changed, 23 insertions(+), 9 deletions(-) diff --git a/tests/test_model/test_v0_4.py b/tests/test_model/test_v0_4.py index 3e00ac3a0..acf5e7e40 100644 --- a/tests/test_model/test_v0_4.py +++ b/tests/test_model/test_v0_4.py @@ -307,7 +307,7 @@ def model_data(): return ModelDescr( documentation=UNET2D_ROOT / "README.md", license="MIT", - git_repo="https://github.com/bioimage-io/python-bioimage-io", + git_repo="https://github.com/bioimage-io/core-bioimage-io-python", description="description", authors=[ Author(name="Author 1", affiliation="Affiliation 1"), diff --git a/tests/test_model/test_v0_5.py b/tests/test_model/test_v0_5.py index c472e2c08..eeded455d 100644 --- a/tests/test_model/test_v0_5.py +++ b/tests/test_model/test_v0_5.py @@ -211,7 +211,7 @@ def model_data(): model = ModelDescr( documentation=UNET2D_ROOT / "README.md", license=LicenseId("MIT"), - git_repo=HttpUrl("https://github.com/bioimage-io/python-bioimage-io"), + git_repo=HttpUrl("https://github.com/bioimage-io/core-bioimage-io-python"), format_version="0.5.0", description="description", authors=[ @@ -340,7 +340,9 @@ def test_warn_long_name(model_data: Dict[str, Any]): model_data["name"] = ( "veeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeery loooooooooooooooong name" ) - summary = validate_format(model_data) + summary = validate_format( + model_data, context=ValidationContext(perform_io_checks=False) + ) assert summary.status == "passed", summary.format() assert summary.details[1].warnings[0].loc == ("name",), summary.format() @@ -349,13 +351,17 @@ def test_warn_long_name(model_data: Dict[str, Any]): def test_model_schema_raises_invalid_input_id(model_data: Dict[str, Any]): model_data["inputs"][0]["id"] = "invalid/id" - summary = validate_format(model_data) + summary = validate_format( + model_data, context=ValidationContext(perform_io_checks=False) + ) assert summary.status == "failed", summary.format() def test_output_fixed_shape_too_small(model_data: Dict[str, Any]): model_data["outputs"][0]["halo"] = 999 - summary = validate_format(model_data) + summary = validate_format( + model_data, context=ValidationContext(perform_io_checks=False) + ) assert summary.status == "failed", summary.format() @@ -393,7 +399,9 @@ def test_output_ref_shape_mismatch(model_data: Dict[str, Any]): "size": {"tensor_id": "input_1", "axis_id": "x"}, "halo": 2, } - summary = validate_format(model_data) + summary = validate_format( + model_data, context=ValidationContext(perform_io_checks=False) + ) assert summary.status == "passed", summary.format() # input_1.x -> input_1.z model_data["outputs"][0]["axes"][2] = { @@ -429,7 +437,9 @@ def test_output_ref_shape_too_small(model_data: Dict[str, Any]): def test_model_has_parent_with_id(model_data: Dict[str, Any]): model_data["parent"] = dict(id="10.5281/zenodo.5764892", version_number=1) - summary = validate_format(model_data) + summary = validate_format( + model_data, context=ValidationContext(perform_io_checks=False) + ) assert summary.status == "passed", summary.format() @@ -450,11 +460,15 @@ def test_model_with_expanded_output(model_data: Dict[str, Any]): def test_model_rdf_is_valid_general_rdf(model_data: Dict[str, Any]): model_data["type"] = "model_as_generic" model_data["format_version"] = "0.3.0" - summary = validate_format(model_data) + summary = validate_format( + model_data, context=ValidationContext(perform_io_checks=False) + ) assert summary.status == "passed", summary.format() def test_model_does_not_accept_unknown_fields(model_data: Dict[str, Any]): model_data["unknown_additional_field"] = "shouldn't be here" - summary = validate_format(model_data) + summary = validate_format( + model_data, context=ValidationContext(perform_io_checks=False) + ) assert summary.status == "failed", summary.format() From d777407d0466db353c1f91957b57c54686628bab Mon Sep 17 00:00:00 2001 From: fynnbe Date: Tue, 19 Mar 2024 14:23:07 +0100 Subject: [PATCH 22/36] fix license unions --- bioimageio/spec/generic/v0_2.py | 6 ++++-- bioimageio/spec/generic/v0_3.py | 4 +++- tests/test_internal/test_license_id.py | 18 +++++++++++++++++- 3 files changed, 24 insertions(+), 4 deletions(-) diff --git a/bioimageio/spec/generic/v0_2.py b/bioimageio/spec/generic/v0_2.py index 9765c6a76..9405e797f 100644 --- a/bioimageio/spec/generic/v0_2.py +++ b/bioimageio/spec/generic/v0_2.py @@ -369,8 +369,10 @@ def _convert_from_older_format( The recommended documentation file name is `README.md`. An `.md` suffix is mandatory.""" license: Annotated[ - Optional[Union[LicenseId, DeprecatedLicenseId, str]], - Field(examples=["CC-BY-4.0", "MIT", "BSD-2-Clause"]), + Union[LicenseId, DeprecatedLicenseId, str, None], + Field( + union_mode="left_to_right", examples=["CC-BY-4.0", "MIT", "BSD-2-Clause"] + ), ] = None """A [SPDX license identifier](https://spdx.org/licenses/). We do not support custom license beyond the SPDX license list, if you need that please diff --git a/bioimageio/spec/generic/v0_3.py b/bioimageio/spec/generic/v0_3.py index 2ecc4b578..d8d443366 100644 --- a/bioimageio/spec/generic/v0_3.py +++ b/bioimageio/spec/generic/v0_3.py @@ -206,7 +206,9 @@ class GenericModelDescrBase(ResourceDescrBase): """citations""" license: Annotated[ - Union[LicenseId, DeprecatedLicenseId], + Annotated[ + Union[LicenseId, DeprecatedLicenseId], Field(union_mode="left_to_right") + ], warn( LicenseId, "{value} is deprecated, see https://spdx.org/licenses/{value}.html", diff --git a/tests/test_internal/test_license_id.py b/tests/test_internal/test_license_id.py index 57e1721b3..f851cb345 100644 --- a/tests/test_internal/test_license_id.py +++ b/tests/test_internal/test_license_id.py @@ -1,5 +1,7 @@ +from typing import Union + import pytest -from pydantic import ValidationError +from pydantic import BaseModel, Field, ValidationError def test_license_id(): @@ -11,6 +13,20 @@ def test_license_id(): _ = LicenseId("not_a_valid_license_id") # pyright: ignore[reportArgumentType] +def test_license_id_in_model(): + """pydantic 2 now defaults to smart unions, which try to find the best match, + somehow `str` is considered a better match than a valid `LicneseId`, so we should + only use 'left_to_right' unions with `RootModel[str]`, e.g. `ValidatedString`""" + from bioimageio.spec._internal.license_id import LicenseId + + class Model(BaseModel): + lid: Union[LicenseId, str] = Field(union_mode="left_to_right") + + out = Model.model_validate({"lid": "CC-BY-4.0"}).lid + assert isinstance(out, LicenseId) + assert not isinstance(out, str) + + def test_deprecated_license_id(): from bioimageio.spec._internal.license_id import DeprecatedLicenseId From 6cb2cf7c14268da2783157f28ee9102b35451e46 Mon Sep 17 00:00:00 2001 From: fynnbe Date: Tue, 19 Mar 2024 18:11:17 +0100 Subject: [PATCH 23/36] make ValidatedString inherite from str --- bioimageio/spec/_internal/io.py | 13 +++-- bioimageio/spec/_internal/license_id.py | 19 ++----- bioimageio/spec/_internal/root_url.py | 27 +++------- bioimageio/spec/_internal/types.py | 53 +++++++++++++------ bioimageio/spec/_internal/url.py | 22 ++++---- bioimageio/spec/_internal/validated_string.py | 28 +++++++--- bioimageio/spec/application/v0_2.py | 7 +-- bioimageio/spec/application/v0_3.py | 8 +-- bioimageio/spec/collection/v0_2.py | 11 ++-- bioimageio/spec/collection/v0_3.py | 10 ++-- bioimageio/spec/dataset/v0_2.py | 8 +-- bioimageio/spec/dataset/v0_3.py | 8 +-- bioimageio/spec/generic/v0_2.py | 29 +++++++--- bioimageio/spec/generic/v0_3.py | 18 ++++--- bioimageio/spec/model/v0_4.py | 20 +++---- bioimageio/spec/model/v0_5.py | 37 ++++++------- bioimageio/spec/notebook/v0_2.py | 9 ++-- bioimageio/spec/notebook/v0_3.py | 7 +-- tests/test_internal/test_license_id.py | 6 +-- tests/test_internal/test_validated_string.py | 18 +++++++ tests/test_specific_reexports_generics.py | 3 +- 21 files changed, 208 insertions(+), 153 deletions(-) create mode 100644 tests/test_internal/test_validated_string.py diff --git a/bioimageio/spec/_internal/io.py b/bioimageio/spec/_internal/io.py index fc66e9fe0..9689b468b 100644 --- a/bioimageio/spec/_internal/io.py +++ b/bioimageio/spec/_internal/io.py @@ -73,18 +73,17 @@ SLOTS = {"slots": True} -class Sha256( - ValidatedString[ +class Sha256(ValidatedString): + """SHA-256 hash value""" + + root_model = RootModel[ Annotated[ str, StringConstraints( strip_whitespace=True, to_lower=True, min_length=64, max_length=64 ), ] - ], - frozen=True, -): - """SHA-256 hash value""" + ] AbsolutePathT = TypeVar( @@ -323,7 +322,7 @@ def _package(value: FileSource, info: SerializationInfo) -> Union[str, Path, Fil if isinstance(value, Path): unpackaged = value elif isinstance(value, HttpUrl): - unpackaged = value.root + unpackaged = value elif isinstance(value, RelativeFilePath): unpackaged = Path(value.path) elif isinstance(value, AnyUrl): diff --git a/bioimageio/spec/_internal/license_id.py b/bioimageio/spec/_internal/license_id.py index 0e3dff689..ece095492 100644 --- a/bioimageio/spec/_internal/license_id.py +++ b/bioimageio/spec/_internal/license_id.py @@ -1,4 +1,4 @@ -from typing import TypeVar +from pydantic import RootModel from ._generated_spdx_license_literals import ( DeprecatedLicenseId as DeprecatedLicenseIdLiteral, @@ -6,19 +6,10 @@ from ._generated_spdx_license_literals import LicenseId as LicenseIdLiteral from .validated_string import ValidatedString -LicenceT = TypeVar("LicenceT", LicenseIdLiteral, DeprecatedLicenseIdLiteral) +class LicenseId(ValidatedString): + root_model = RootModel[LicenseIdLiteral] -class _LicenseId(ValidatedString[LicenceT], frozen=True): - def __repr__(self): - # don't include full literal in class repr - name, *_ = self.__class__.__name__.split("[") - return f'{name}("{self.root}")' - -class DeprecatedLicenseId(_LicenseId[DeprecatedLicenseIdLiteral], frozen=True): - pass - - -class LicenseId(_LicenseId[LicenseIdLiteral], frozen=True): - pass +class DeprecatedLicenseId(ValidatedString): + root_model = RootModel[DeprecatedLicenseIdLiteral] diff --git a/bioimageio/spec/_internal/root_url.py b/bioimageio/spec/_internal/root_url.py index 9f9537da9..ed2d0c86c 100644 --- a/bioimageio/spec/_internal/root_url.py +++ b/bioimageio/spec/_internal/root_url.py @@ -4,41 +4,28 @@ from urllib.parse import urlsplit, urlunsplit import pydantic -from pydantic import TypeAdapter -from typing_extensions import Annotated +from pydantic import RootModel from .validated_string import ValidatedString -from .validator_annotations import AfterValidator -_http_url_adapter = TypeAdapter(pydantic.HttpUrl) # pyright: ignore[reportCallIssue] - -class RootHttpUrl( - ValidatedString[ - Annotated[ - str, - AfterValidator(lambda value: str(_http_url_adapter.validate_python(value))), - ] - ], - frozen=True, -): +class RootHttpUrl(ValidatedString): """A 'URL folder', possibly an invalid http URL""" - @property - def _url(self): - return pydantic.AnyUrl(str(self)) + root_model = RootModel[pydantic.HttpUrl] + _validated: pydantic.HttpUrl @property def scheme(self) -> str: - return self._url.scheme + return self._validated.scheme @property def host(self) -> Optional[str]: - return self._url.host + return self._validated.host @property def path(self) -> Optional[str]: - return self._url.path + return self._validated.path @property def parent(self) -> RootHttpUrl: diff --git a/bioimageio/spec/_internal/types.py b/bioimageio/spec/_internal/types.py index 07dd8aadf..6547043cc 100644 --- a/bioimageio/spec/_internal/types.py +++ b/bioimageio/spec/_internal/types.py @@ -90,27 +90,48 @@ class Datetime( """ -Doi = ValidatedString[Annotated[str, StringConstraints(pattern=DOI_REGEX)]] +class Doi(ValidatedString): + root_model = RootModel[Annotated[str, StringConstraints(pattern=DOI_REGEX)]] + + FormatVersionPlaceholder = Literal["latest", "discover"] IdentifierAnno = Annotated[ NotEmpty[str], AfterValidator(_validate_identifier), AfterValidator(_validate_is_not_keyword), ] -Identifier = ValidatedString[IdentifierAnno] + + +class Identifier(ValidatedString): + root_model = RootModel[IdentifierAnno] + + LowerCaseIdentifierAnno = Annotated[IdentifierAnno, annotated_types.LowerCase] -LowerCaseIdentifier = ValidatedString[LowerCaseIdentifierAnno] -OrcidId = ValidatedString[Annotated[str, AfterValidator(_validate_orcid_id)]] -SiUnit = ValidatedString[ - Annotated[ - str, - StringConstraints(min_length=1, pattern=SI_UNIT_REGEX), - BeforeValidator( - lambda s: ( - s.replace("×", "·").replace("*", "·").replace(" ", "·") - if isinstance(s, str) - else s - ) - ), + + +class LowerCaseIdentifier(ValidatedString): + root_model = RootModel[LowerCaseIdentifierAnno] + + +class OrcidId(ValidatedString): + """an ORCID identifier""" + + root_model = RootModel[Annotated[str, AfterValidator(_validate_orcid_id)]] + + +class SiUnit(ValidatedString): + """Si unit""" + + root_model = RootModel[ + Annotated[ + str, + StringConstraints(min_length=1, pattern=SI_UNIT_REGEX), + BeforeValidator( + lambda s: ( + s.replace("×", "·").replace("*", "·").replace(" ", "·") + if isinstance(s, str) + else s + ) + ), + ] ] -] diff --git a/bioimageio/spec/_internal/url.py b/bioimageio/spec/_internal/url.py index 11728b78e..8ecb47b3c 100644 --- a/bioimageio/spec/_internal/url.py +++ b/bioimageio/spec/_internal/url.py @@ -1,13 +1,21 @@ +from typing import Union + +import pydantic import requests import requests.exceptions -from pydantic import model_validator +from pydantic import AfterValidator, RootModel +from typing_extensions import Annotated from .field_warning import issue_warning from .root_url import RootHttpUrl from .validation_context import validation_context_var -def check_url(url: str) -> None: +def _validate_url(url: Union[str, pydantic.HttpUrl]) -> pydantic.AnyUrl: + url = str(url) + if not validation_context_var.get().perform_io_checks: + return pydantic.AnyUrl(url) + if url.startswith("https://colab.research.google.com/github/"): # head request for colab returns "Value error, 405: Method Not Allowed" # therefore we check if the source notebook exists at github instead @@ -88,12 +96,8 @@ def check_url(url: str) -> None: # TODO follow up forbidden head request with get # motivating example: 403: Forbidden https://elifesciences.org/articles/57613 + return pydantic.AnyUrl(url) -class HttpUrl(RootHttpUrl, frozen=True): - @model_validator(mode="after") - def _check_url(self): - if not validation_context_var.get().perform_io_checks: - return self - check_url(str(self)) - return self +class HttpUrl(RootHttpUrl): + root_model = RootModel[Annotated[pydantic.HttpUrl, AfterValidator(_validate_url)]] diff --git a/bioimageio/spec/_internal/validated_string.py b/bioimageio/spec/_internal/validated_string.py index f635c69c8..cb0666e43 100644 --- a/bioimageio/spec/_internal/validated_string.py +++ b/bioimageio/spec/_internal/validated_string.py @@ -1,10 +1,26 @@ -from typing import TypeVar +from typing import Any, ClassVar, Type -from pydantic import RootModel +from pydantic import GetCoreSchemaHandler, RootModel +from pydantic_core.core_schema import ( + CoreSchema, + no_info_after_validator_function, +) -S = TypeVar("S", bound=str) +class ValidatedString(str): + root_model: ClassVar[Type[RootModel[Any]]] = RootModel[str] + """the pydantic root model to validate the string""" + # TODO: should we use a TypeAdapter instead? + # TODO: with future py version: RootModel[Any] -> RootModel[str | "literal string type"] + _validated: Any # pyright: ignore[reportUninitializedInstanceVariable] # initalized in __new__ -class ValidatedString(RootModel[S], frozen=True): - def __str__(self) -> str: - return self.root + def __new__(cls, object: object): + self = super().__new__(cls, object) + self._validated = cls.root_model.model_validate(self) + return self + + @classmethod + def __get_pydantic_core_schema__( + cls, source_type: Any, handler: GetCoreSchemaHandler + ) -> CoreSchema: + return no_info_after_validator_function(cls, handler(str)) diff --git a/bioimageio/spec/application/v0_2.py b/bioimageio/spec/application/v0_2.py index 3e799437d..a0e9348c8 100644 --- a/bioimageio/spec/application/v0_2.py +++ b/bioimageio/spec/application/v0_2.py @@ -7,13 +7,12 @@ from .._internal.io_basics import AbsoluteFilePath as AbsoluteFilePath from .._internal.types import ImportantFileSource from .._internal.url import HttpUrl as HttpUrl -from .._internal.validated_string import ValidatedString from ..generic.v0_2 import AttachmentsDescr as AttachmentsDescr from ..generic.v0_2 import Author as Author from ..generic.v0_2 import BadgeDescr as BadgeDescr from ..generic.v0_2 import CiteEntry as CiteEntry from ..generic.v0_2 import Doi as Doi -from ..generic.v0_2 import GenericDescrBase, ResourceId_v0_2_Anno +from ..generic.v0_2 import GenericDescrBase from ..generic.v0_2 import LinkedResource as LinkedResource from ..generic.v0_2 import Maintainer as Maintainer from ..generic.v0_2 import OrcidId as OrcidId @@ -22,7 +21,9 @@ from ..generic.v0_2 import Uploader as Uploader from ..generic.v0_2 import Version as Version -ApplicationId = ValidatedString[ResourceId_v0_2_Anno] + +class ApplicationId(ResourceId): + pass class ApplicationDescr(GenericDescrBase, title="bioimage.io application specification"): diff --git a/bioimageio/spec/application/v0_3.py b/bioimageio/spec/application/v0_3.py index acc27830a..31c39b7b2 100644 --- a/bioimageio/spec/application/v0_3.py +++ b/bioimageio/spec/application/v0_3.py @@ -9,21 +9,21 @@ from .._internal.io_basics import AbsoluteFilePath as AbsoluteFilePath from .._internal.types import ImportantFileSource from .._internal.url import HttpUrl as HttpUrl -from .._internal.validated_string import ValidatedString from ..generic.v0_3 import Author as Author from ..generic.v0_3 import BadgeDescr as BadgeDescr from ..generic.v0_3 import CiteEntry as CiteEntry from ..generic.v0_3 import Doi as Doi -from ..generic.v0_3 import GenericDescrBase, ResourceIdAnno +from ..generic.v0_3 import GenericDescrBase, ResourceId from ..generic.v0_3 import LinkedResource as LinkedResource from ..generic.v0_3 import Maintainer as Maintainer from ..generic.v0_3 import OrcidId as OrcidId from ..generic.v0_3 import RelativeFilePath as RelativeFilePath -from ..generic.v0_3 import ResourceId as ResourceId from ..generic.v0_3 import Uploader as Uploader from ..generic.v0_3 import Version as Version -ApplicationId = ValidatedString[ResourceIdAnno] + +class ApplicationId(ResourceId): + pass class ApplicationDescr(GenericDescrBase, title="bioimage.io application specification"): diff --git a/bioimageio/spec/collection/v0_2.py b/bioimageio/spec/collection/v0_2.py index 763e699c3..3a4d10588 100644 --- a/bioimageio/spec/collection/v0_2.py +++ b/bioimageio/spec/collection/v0_2.py @@ -17,7 +17,6 @@ from .._internal.io_utils import open_bioimageio_yaml from .._internal.types import NotEmpty from .._internal.url import HttpUrl as HttpUrl -from .._internal.validated_string import ValidatedString from .._internal.validation_context import validation_context_var from .._internal.warning_levels import ALERT from ..application import ApplicationDescr_v0_2, ApplicationDescr_v0_3 @@ -28,17 +27,21 @@ from ..generic.v0_2 import BadgeDescr as BadgeDescr from ..generic.v0_2 import CiteEntry as CiteEntry from ..generic.v0_2 import Doi as Doi -from ..generic.v0_2 import FileSource, GenericDescrBase, ResourceId_v0_2_Anno +from ..generic.v0_2 import FileSource, GenericDescrBase from ..generic.v0_2 import LinkedResource as LinkedResource from ..generic.v0_2 import Maintainer as Maintainer from ..generic.v0_2 import OrcidId as OrcidId from ..generic.v0_2 import RelativeFilePath as RelativeFilePath +from ..generic.v0_2 import ResourceId as ResourceId from ..generic.v0_2 import Uploader as Uploader from ..generic.v0_2 import Version as Version from ..model import ModelDescr_v0_4, ModelDescr_v0_5 from ..notebook import NotebookDescr_v0_2, NotebookDescr_v0_3 -CollectionId = ValidatedString[ResourceId_v0_2_Anno] + +class CollectionId(ResourceId): + pass + EntryDescr = Union[ ApplicationDescr_v0_2, @@ -110,7 +113,7 @@ class CollectionEntry(Node, extra="allow"): rdf_source: Optional[FileSource] = None """resource description file (RDF) source to load entry from""" - id: Optional[ResourceId_v0_2_Anno] = None + id: Optional[ResourceId] = None """Collection entry sub id overwriting `rdf_source.id`. The full collection entry's id is the collection's base id, followed by this sub id and separated by a slash '/'.""" diff --git a/bioimageio/spec/collection/v0_3.py b/bioimageio/spec/collection/v0_3.py index 9b77aecba..f011a9713 100644 --- a/bioimageio/spec/collection/v0_3.py +++ b/bioimageio/spec/collection/v0_3.py @@ -25,7 +25,6 @@ from .._internal.io_utils import open_bioimageio_yaml from .._internal.types import FileSource, NotEmpty from .._internal.url import HttpUrl as HttpUrl -from .._internal.validated_string import ValidatedString from .._internal.validation_context import ( validation_context_var, ) @@ -39,7 +38,7 @@ from ..generic.v0_3 import Doi as Doi from ..generic.v0_3 import ( GenericDescrBase, - ResourceIdAnno, + ResourceId, _author_conv, # pyright: ignore[reportPrivateUsage] _maintainer_conv, # pyright: ignore[reportPrivateUsage] ) @@ -53,7 +52,10 @@ from ..notebook import NotebookDescr_v0_2, NotebookDescr_v0_3 from .v0_2 import CollectionDescr as _CollectionDescr_v0_2 -CollectionId = ValidatedString[ResourceIdAnno] + +class CollectionId(ResourceId): + pass + EntryDescr = Union[ ApplicationDescr_v0_2, @@ -129,7 +131,7 @@ class CollectionEntry(Node, extra="allow"): entry_source: Optional[FileSource] = None """an external source this entry description is based on""" - id: Optional[ResourceIdAnno] = None + id: Optional[ResourceId] = None """Collection entry sub id overwriting `rdf_source.id`. The full collection entry's id is the collection's base id, followed by this sub id and separated by a slash '/'.""" diff --git a/bioimageio/spec/dataset/v0_2.py b/bioimageio/spec/dataset/v0_2.py index 4adb34535..ca1a3ad73 100644 --- a/bioimageio/spec/dataset/v0_2.py +++ b/bioimageio/spec/dataset/v0_2.py @@ -3,22 +3,22 @@ from .._internal.common_nodes import Node from .._internal.io_basics import AbsoluteFilePath as AbsoluteFilePath from .._internal.url import HttpUrl as HttpUrl -from .._internal.validated_string import ValidatedString from ..generic.v0_2 import AttachmentsDescr as AttachmentsDescr from ..generic.v0_2 import Author as Author from ..generic.v0_2 import BadgeDescr as BadgeDescr from ..generic.v0_2 import CiteEntry as CiteEntry from ..generic.v0_2 import Doi as Doi -from ..generic.v0_2 import GenericDescrBase, ResourceId_v0_2_Anno +from ..generic.v0_2 import GenericDescrBase, ResourceId from ..generic.v0_2 import LinkedResource as LinkedResource from ..generic.v0_2 import Maintainer as Maintainer from ..generic.v0_2 import OrcidId as OrcidId from ..generic.v0_2 import RelativeFilePath as RelativeFilePath -from ..generic.v0_2 import ResourceId as ResourceId from ..generic.v0_2 import Uploader as Uploader from ..generic.v0_2 import Version as Version -DatasetId = ValidatedString[ResourceId_v0_2_Anno] + +class DatasetId(ResourceId): + pass class DatasetDescr(GenericDescrBase, title="bioimage.io dataset specification"): diff --git a/bioimageio/spec/dataset/v0_3.py b/bioimageio/spec/dataset/v0_3.py index 79922eaec..39c183a4e 100644 --- a/bioimageio/spec/dataset/v0_3.py +++ b/bioimageio/spec/dataset/v0_3.py @@ -7,14 +7,12 @@ from .._internal.io import Sha256 as Sha256 from .._internal.io_basics import AbsoluteFilePath as AbsoluteFilePath from .._internal.url import HttpUrl as HttpUrl -from .._internal.validated_string import ValidatedString from ..generic.v0_3 import Author as Author from ..generic.v0_3 import BadgeDescr as BadgeDescr from ..generic.v0_3 import CiteEntry as CiteEntry from ..generic.v0_3 import ( DocumentationSource, GenericDescrBase, - ResourceIdAnno, _author_conv, # pyright: ignore[reportPrivateUsage] _maintainer_conv, # pyright: ignore[reportPrivateUsage] ) @@ -28,7 +26,9 @@ from ..generic.v0_3 import Version as Version from .v0_2 import DatasetDescr as DatasetDescr02 -DatasetId = ValidatedString[ResourceIdAnno] + +class DatasetId(ResourceId): + pass class DatasetDescr(GenericDescrBase, title="bioimage.io dataset specification"): @@ -81,7 +81,7 @@ def _convert(cls, data: Dict[str, Any], /) -> Dict[str, Any]: format_version="0.3.0", git_repo=old.git_repo, # pyright: ignore[reportArgumentType] icon=old.icon, - id=old.id, + id=None if old.id is None else DatasetId(old.id), license=old.license, # type: ignore links=old.links, maintainers=[ diff --git a/bioimageio/spec/generic/v0_2.py b/bioimageio/spec/generic/v0_2.py index 36a3c75cb..436899156 100644 --- a/bioimageio/spec/generic/v0_2.py +++ b/bioimageio/spec/generic/v0_2.py @@ -14,7 +14,14 @@ import annotated_types from annotated_types import Len, LowerCase, MaxLen -from pydantic import EmailStr, Field, ValidationInfo, field_validator, model_validator +from pydantic import ( + EmailStr, + Field, + RootModel, + ValidationInfo, + field_validator, + model_validator, +) from typing_extensions import Annotated, Self, assert_never from .._internal.common_nodes import Node, ResourceDescrBase @@ -43,13 +50,19 @@ from .._internal.version_type import Version as Version from ._v0_2_converter import convert_from_older_format as _convert_from_older_format -ResourceId_v0_2_Anno = Annotated[ - NotEmpty[str], - AfterValidator(lambda s: s.lower()), # convert upper case on the fly - RestrictCharacters(string.ascii_lowercase + string.digits + "_-/."), - annotated_types.Predicate(lambda s: not (s.startswith("/") or s.endswith("/"))), -] -ResourceId = ValidatedString[ResourceId_v0_2_Anno] + +class ResourceId(ValidatedString): + root_model = RootModel[ + Annotated[ + NotEmpty[str], + AfterValidator(str.lower), # convert upper case on the fly + RestrictCharacters(string.ascii_lowercase + string.digits + "_-/."), + annotated_types.Predicate( + lambda s: not (s.startswith("/") or s.endswith("/")) + ), + ] + ] + KNOWN_SPECIFIC_RESOURCE_TYPES = ( "application", diff --git a/bioimageio/spec/generic/v0_3.py b/bioimageio/spec/generic/v0_3.py index 0a648ebdc..8eed0e542 100644 --- a/bioimageio/spec/generic/v0_3.py +++ b/bioimageio/spec/generic/v0_3.py @@ -6,7 +6,7 @@ import annotated_types from annotated_types import Len, LowerCase, MaxLen, MinLen -from pydantic import Field, ValidationInfo, field_validator, model_validator +from pydantic import Field, RootModel, ValidationInfo, field_validator, model_validator from typing_extensions import Annotated from bioimageio.spec._internal.field_validation import validate_gh_user @@ -63,13 +63,17 @@ "notebook", ) -ResourceIdAnno = Annotated[ - NotEmpty[str], - RestrictCharacters(string.ascii_lowercase + string.digits + "_-/."), - annotated_types.Predicate(lambda s: not (s.startswith("/") or s.endswith("/"))), -] -ResourceId = ValidatedString[ResourceIdAnno] +class ResourceId(ValidatedString): + root_model = RootModel[ + Annotated[ + NotEmpty[str], + RestrictCharacters(string.ascii_lowercase + string.digits + "_-/."), + annotated_types.Predicate( + lambda s: not (s.startswith("/") or s.endswith("/")) + ), + ] + ] def _validate_md_suffix(value: V_suffix) -> V_suffix: diff --git a/bioimageio/spec/model/v0_4.py b/bioimageio/spec/model/v0_4.py index 92941fc2c..e1316ab68 100644 --- a/bioimageio/spec/model/v0_4.py +++ b/bioimageio/spec/model/v0_4.py @@ -27,8 +27,6 @@ ) from typing_extensions import Annotated, LiteralString, Self, assert_never -from bioimageio.spec._internal.validated_string import ValidatedString - from .._internal.common_nodes import ( KwargsNode, Node, @@ -49,10 +47,7 @@ from .._internal.io_basics import AbsoluteFilePath as AbsoluteFilePath from .._internal.types import Datetime as Datetime from .._internal.types import Identifier as Identifier -from .._internal.types import ( - ImportantFileSource, - LowerCaseIdentifierAnno, -) +from .._internal.types import ImportantFileSource, LowerCaseIdentifier from .._internal.types import LicenseId as LicenseId from .._internal.types import NotEmpty as NotEmpty from .._internal.url import HttpUrl as HttpUrl @@ -66,17 +61,20 @@ from ..generic.v0_2 import BadgeDescr as BadgeDescr from ..generic.v0_2 import CiteEntry as CiteEntry from ..generic.v0_2 import Doi as Doi -from ..generic.v0_2 import GenericModelDescrBase, ResourceId_v0_2_Anno +from ..generic.v0_2 import GenericModelDescrBase from ..generic.v0_2 import LinkedResource as LinkedResource from ..generic.v0_2 import Maintainer as Maintainer from ..generic.v0_2 import OrcidId as OrcidId from ..generic.v0_2 import RelativeFilePath as RelativeFilePath +from ..generic.v0_2 import ResourceId as ResourceId from ..generic.v0_2 import Uploader as Uploader -from ..generic.v0_3 import ResourceId as ResourceId from ..utils import load_array from ._v0_4_converter import convert_from_older_format -ModelId = ValidatedString[ResourceId_v0_2_Anno] + +class ModelId(ResourceId): + pass + AxesStr = Annotated[ str, RestrictCharacters("bitczyx"), AfterValidator(validate_unique_entries) @@ -103,7 +101,9 @@ "scale_range", ] -TensorName = ValidatedString[LowerCaseIdentifierAnno] + +class TensorName(LowerCaseIdentifier): + pass class CallableFromDepencency(StringNode): diff --git a/bioimageio/spec/model/v0_5.py b/bioimageio/spec/model/v0_5.py index bf7788416..9d209c0a4 100644 --- a/bioimageio/spec/model/v0_5.py +++ b/bioimageio/spec/model/v0_5.py @@ -38,20 +38,15 @@ from numpy.typing import NDArray from pydantic import ( Field, - GetCoreSchemaHandler, + RootModel, ValidationInfo, field_validator, model_validator, ) -from pydantic_core.core_schema import ( - CoreSchema, - no_info_after_validator_function, -) from typing_extensions import Annotated, LiteralString, Self, assert_never -from bioimageio.spec._internal.validated_string import ValidatedString from bioimageio.spec._internal.validator_annotations import RestrictCharacters -from bioimageio.spec.generic.v0_3 import ResourceIdAnno +from bioimageio.spec.generic.v0_3 import ResourceId as ResourceId from .._internal.common_nodes import ( Converter, @@ -70,7 +65,12 @@ from .._internal.types import Datetime as Datetime from .._internal.types import DeprecatedLicenseId as DeprecatedLicenseId from .._internal.types import Identifier as Identifier -from .._internal.types import ImportantFileSource, LowerCaseIdentifierAnno, SiUnit +from .._internal.types import ( + ImportantFileSource, + LowerCaseIdentifier, + LowerCaseIdentifierAnno, + SiUnit, +) from .._internal.types import LicenseId as LicenseId from .._internal.types import NotEmpty as NotEmpty from .._internal.url import HttpUrl as HttpUrl @@ -177,22 +177,14 @@ ] AxisType = Literal["batch", "channel", "index", "time", "space"] -TensorId = ValidatedString[Annotated[LowerCaseIdentifierAnno, MaxLen(32)]] -class AxisId(str): - root_model = ValidatedString[Annotated[LowerCaseIdentifierAnno, MaxLen(16)]] +class TensorId(LowerCaseIdentifier): + root_model = RootModel[Annotated[LowerCaseIdentifierAnno, MaxLen(32)]] - @classmethod - def __get_pydantic_core_schema__( - cls, source_type: Any, handler: GetCoreSchemaHandler - ) -> CoreSchema: - return no_info_after_validator_function(cls._validate, handler(str)) - @classmethod - def _validate(cls, value: str): - valid = cls.root_model.model_validate(value) - return cls(valid.root) +class AxisId(LowerCaseIdentifier): + root_model = RootModel[Annotated[LowerCaseIdentifierAnno, MaxLen(16)]] NonBatchAxisId = Annotated[AxisId, Predicate(lambda x: x != "batch")] @@ -1818,7 +1810,8 @@ def check_entries(self) -> Self: return self -ModelId = ValidatedString[ResourceIdAnno] +class ModelId(ResourceId): + pass class LinkedModel(Node): @@ -2349,7 +2342,7 @@ def conv_authors(auths: Optional[Sequence[_Author_v0_4]]): format_version="0.5.0", git_repo=src.git_repo, # pyright: ignore[reportArgumentType] icon=src.icon, - id=src.id, + id=None if src.id is None else ModelId(src.id), id_emoji=src.id_emoji, license=src.license, # type: ignore links=src.links, diff --git a/bioimageio/spec/notebook/v0_2.py b/bioimageio/spec/notebook/v0_2.py index 21b564188..9414c912d 100644 --- a/bioimageio/spec/notebook/v0_2.py +++ b/bioimageio/spec/notebook/v0_2.py @@ -6,21 +6,24 @@ from .._internal.io import WithSuffix from .._internal.io_basics import AbsoluteFilePath as AbsoluteFilePath from .._internal.url import HttpUrl -from .._internal.validated_string import ValidatedString from ..generic.v0_2 import AttachmentsDescr as AttachmentsDescr from ..generic.v0_2 import Author as Author from ..generic.v0_2 import BadgeDescr as BadgeDescr from ..generic.v0_2 import CiteEntry as CiteEntry from ..generic.v0_2 import Doi as Doi -from ..generic.v0_2 import GenericDescrBase, ResourceId_v0_2_Anno +from ..generic.v0_2 import GenericDescrBase from ..generic.v0_2 import LinkedResource as LinkedResource from ..generic.v0_2 import Maintainer as Maintainer from ..generic.v0_2 import OrcidId as OrcidId from ..generic.v0_2 import RelativeFilePath as RelativeFilePath +from ..generic.v0_2 import ResourceId as ResourceId from ..generic.v0_2 import Uploader as Uploader from ..generic.v0_2 import Version as Version -NotebookId = ValidatedString[ResourceId_v0_2_Anno] + +class NotebookId(ResourceId): + pass + _WithNotebookSuffix = WithSuffix(".ipynb", case_sensitive=True) NotebookSource = Union[ diff --git a/bioimageio/spec/notebook/v0_3.py b/bioimageio/spec/notebook/v0_3.py index 86114a086..020b70e63 100644 --- a/bioimageio/spec/notebook/v0_3.py +++ b/bioimageio/spec/notebook/v0_3.py @@ -5,12 +5,11 @@ from .._internal.io import Sha256 as Sha256 from .._internal.io_basics import AbsoluteFilePath as AbsoluteFilePath from .._internal.url import HttpUrl as HttpUrl -from .._internal.validated_string import ValidatedString from ..generic.v0_3 import Author as Author from ..generic.v0_3 import BadgeDescr as BadgeDescr from ..generic.v0_3 import CiteEntry as CiteEntry from ..generic.v0_3 import Doi as Doi -from ..generic.v0_3 import GenericDescrBase, ResourceIdAnno +from ..generic.v0_3 import GenericDescrBase from ..generic.v0_3 import LinkedResource as LinkedResource from ..generic.v0_3 import Maintainer as Maintainer from ..generic.v0_3 import OrcidId as OrcidId @@ -20,7 +19,9 @@ from ..generic.v0_3 import Version as Version from .v0_2 import NotebookSource as NotebookSource -NotebookId = ValidatedString[ResourceIdAnno] + +class NotebookId(ResourceId): + pass class NotebookDescr(GenericDescrBase, title="bioimage.io notebook specification"): diff --git a/tests/test_internal/test_license_id.py b/tests/test_internal/test_license_id.py index 57e1721b3..acbe4556e 100644 --- a/tests/test_internal/test_license_id.py +++ b/tests/test_internal/test_license_id.py @@ -8,7 +8,7 @@ def test_license_id(): _ = LicenseId("MIT") with pytest.raises(ValidationError): - _ = LicenseId("not_a_valid_license_id") # pyright: ignore[reportArgumentType] + _ = LicenseId("not_a_valid_license_id") def test_deprecated_license_id(): @@ -17,6 +17,4 @@ def test_deprecated_license_id(): _ = DeprecatedLicenseId("AGPL-1.0") with pytest.raises(ValidationError): - _ = DeprecatedLicenseId( - "not_a_valid_license_id" # pyright: ignore[reportArgumentType] - ) + _ = DeprecatedLicenseId("not_a_valid_license_id") diff --git a/tests/test_internal/test_validated_string.py b/tests/test_internal/test_validated_string.py new file mode 100644 index 000000000..8be29eaee --- /dev/null +++ b/tests/test_internal/test_validated_string.py @@ -0,0 +1,18 @@ +import annotated_types +import pytest +from pydantic import RootModel, ValidationError +from typing_extensions import Annotated + + +def test_valid_validated_string(): + from bioimageio.spec._internal.validated_string import ValidatedString + + class V(ValidatedString): + root_model = RootModel[Annotated[str, annotated_types.Predicate(str.islower)]] + + out = V("abc") + assert isinstance(out, str) + assert out == "abc" + + with pytest.raises(ValidationError): + _ = V("ABC") diff --git a/tests/test_specific_reexports_generics.py b/tests/test_specific_reexports_generics.py index 4baa2e152..ae9665d69 100644 --- a/tests/test_specific_reexports_generics.py +++ b/tests/test_specific_reexports_generics.py @@ -49,6 +49,7 @@ "Predicate", "requests", "RestrictCharacters", + "RootModel", "Self", "Sequence", "settings", @@ -63,6 +64,7 @@ "v0_5", "validate_gh_user", "validate_suffix", + "ValidatedString", "validation_context_var", "ValidationInfo", "warn", @@ -86,7 +88,6 @@ def get_members(m: ModuleType): "KNOWN_SPECIFIC_RESOURCE_TYPES", "ResourceDescrBase", "ResourceDescrType", - "ResourceId", "VALID_COVER_IMAGE_EXTENSIONS", } From ea406292c774f0c872b0203db19e30f047fd1c26 Mon Sep 17 00:00:00 2001 From: fynnbe Date: Wed, 20 Mar 2024 21:55:00 +0100 Subject: [PATCH 24/36] fix root_model annotations --- bioimageio/spec/_internal/io.py | 3 ++- bioimageio/spec/_internal/license_id.py | 6 ++++-- bioimageio/spec/_internal/root_url.py | 4 ++-- bioimageio/spec/_internal/types.py | 16 ++++++++++------ bioimageio/spec/_internal/url.py | 6 ++++-- bioimageio/spec/generic/v0_2.py | 4 +++- bioimageio/spec/generic/v0_3.py | 15 +++++++++++++-- bioimageio/spec/model/v0_5.py | 8 ++++++-- tests/test_internal/test_validated_string.py | 6 +++++- 9 files changed, 49 insertions(+), 19 deletions(-) diff --git a/bioimageio/spec/_internal/io.py b/bioimageio/spec/_internal/io.py index 9689b468b..847c7c17c 100644 --- a/bioimageio/spec/_internal/io.py +++ b/bioimageio/spec/_internal/io.py @@ -10,6 +10,7 @@ from pathlib import Path, PurePath from typing import ( Any, + ClassVar, Dict, Generic, List, @@ -76,7 +77,7 @@ class Sha256(ValidatedString): """SHA-256 hash value""" - root_model = RootModel[ + root_model: ClassVar[Type[RootModel[Any]]] = RootModel[ Annotated[ str, StringConstraints( diff --git a/bioimageio/spec/_internal/license_id.py b/bioimageio/spec/_internal/license_id.py index ece095492..cef093d17 100644 --- a/bioimageio/spec/_internal/license_id.py +++ b/bioimageio/spec/_internal/license_id.py @@ -1,3 +1,5 @@ +from typing import Any, ClassVar, Type + from pydantic import RootModel from ._generated_spdx_license_literals import ( @@ -8,8 +10,8 @@ class LicenseId(ValidatedString): - root_model = RootModel[LicenseIdLiteral] + root_model: ClassVar[Type[RootModel[Any]]] = RootModel[LicenseIdLiteral] class DeprecatedLicenseId(ValidatedString): - root_model = RootModel[DeprecatedLicenseIdLiteral] + root_model: ClassVar[Type[RootModel[Any]]] = RootModel[DeprecatedLicenseIdLiteral] diff --git a/bioimageio/spec/_internal/root_url.py b/bioimageio/spec/_internal/root_url.py index ed2d0c86c..c88e8ae97 100644 --- a/bioimageio/spec/_internal/root_url.py +++ b/bioimageio/spec/_internal/root_url.py @@ -1,6 +1,6 @@ from __future__ import annotations -from typing import Optional +from typing import Any, ClassVar, Optional, Type from urllib.parse import urlsplit, urlunsplit import pydantic @@ -12,7 +12,7 @@ class RootHttpUrl(ValidatedString): """A 'URL folder', possibly an invalid http URL""" - root_model = RootModel[pydantic.HttpUrl] + root_model: ClassVar[Type[RootModel[Any]]] = RootModel[pydantic.HttpUrl] _validated: pydantic.HttpUrl @property diff --git a/bioimageio/spec/_internal/types.py b/bioimageio/spec/_internal/types.py index 6547043cc..1d8a930da 100644 --- a/bioimageio/spec/_internal/types.py +++ b/bioimageio/spec/_internal/types.py @@ -2,7 +2,7 @@ from datetime import datetime from keyword import iskeyword -from typing import Any, Sequence, TypeVar, Union +from typing import Any, ClassVar, Sequence, Type, TypeVar, Union import annotated_types from dateutil.parser import isoparse @@ -91,7 +91,9 @@ class Datetime( class Doi(ValidatedString): - root_model = RootModel[Annotated[str, StringConstraints(pattern=DOI_REGEX)]] + root_model: ClassVar[Type[RootModel[Any]]] = RootModel[ + Annotated[str, StringConstraints(pattern=DOI_REGEX)] + ] FormatVersionPlaceholder = Literal["latest", "discover"] @@ -103,26 +105,28 @@ class Doi(ValidatedString): class Identifier(ValidatedString): - root_model = RootModel[IdentifierAnno] + root_model: ClassVar[Type[RootModel[Any]]] = RootModel[IdentifierAnno] LowerCaseIdentifierAnno = Annotated[IdentifierAnno, annotated_types.LowerCase] class LowerCaseIdentifier(ValidatedString): - root_model = RootModel[LowerCaseIdentifierAnno] + root_model: ClassVar[Type[RootModel[Any]]] = RootModel[LowerCaseIdentifierAnno] class OrcidId(ValidatedString): """an ORCID identifier""" - root_model = RootModel[Annotated[str, AfterValidator(_validate_orcid_id)]] + root_model: ClassVar[Type[RootModel[Any]]] = RootModel[ + Annotated[str, AfterValidator(_validate_orcid_id)] + ] class SiUnit(ValidatedString): """Si unit""" - root_model = RootModel[ + root_model: ClassVar[Type[RootModel[Any]]] = RootModel[ Annotated[ str, StringConstraints(min_length=1, pattern=SI_UNIT_REGEX), diff --git a/bioimageio/spec/_internal/url.py b/bioimageio/spec/_internal/url.py index 8ecb47b3c..987d429ae 100644 --- a/bioimageio/spec/_internal/url.py +++ b/bioimageio/spec/_internal/url.py @@ -1,4 +1,4 @@ -from typing import Union +from typing import Any, ClassVar, Type, Union import pydantic import requests @@ -100,4 +100,6 @@ def _validate_url(url: Union[str, pydantic.HttpUrl]) -> pydantic.AnyUrl: class HttpUrl(RootHttpUrl): - root_model = RootModel[Annotated[pydantic.HttpUrl, AfterValidator(_validate_url)]] + root_model: ClassVar[Type[RootModel[Any]]] = RootModel[ + Annotated[pydantic.HttpUrl, AfterValidator(_validate_url)] + ] diff --git a/bioimageio/spec/generic/v0_2.py b/bioimageio/spec/generic/v0_2.py index 436899156..b302d3bb0 100644 --- a/bioimageio/spec/generic/v0_2.py +++ b/bioimageio/spec/generic/v0_2.py @@ -2,12 +2,14 @@ import string from typing import ( Any, + ClassVar, Dict, List, Literal, Mapping, Optional, Sequence, + Type, TypeVar, Union, ) @@ -52,7 +54,7 @@ class ResourceId(ValidatedString): - root_model = RootModel[ + root_model: ClassVar[Type[RootModel[Any]]] = RootModel[ Annotated[ NotEmpty[str], AfterValidator(str.lower), # convert upper case on the fly diff --git a/bioimageio/spec/generic/v0_3.py b/bioimageio/spec/generic/v0_3.py index 8eed0e542..58a9a6de0 100644 --- a/bioimageio/spec/generic/v0_3.py +++ b/bioimageio/spec/generic/v0_3.py @@ -2,7 +2,18 @@ import string from functools import partial -from typing import Any, Dict, List, Literal, Optional, Sequence, TypeVar, Union +from typing import ( + Any, + ClassVar, + Dict, + List, + Literal, + Optional, + Sequence, + Type, + TypeVar, + Union, +) import annotated_types from annotated_types import Len, LowerCase, MaxLen, MinLen @@ -65,7 +76,7 @@ class ResourceId(ValidatedString): - root_model = RootModel[ + root_model: ClassVar[Type[RootModel[Any]]] = RootModel[ Annotated[ NotEmpty[str], RestrictCharacters(string.ascii_lowercase + string.digits + "_-/."), diff --git a/bioimageio/spec/model/v0_5.py b/bioimageio/spec/model/v0_5.py index 9d209c0a4..6cf8d5950 100644 --- a/bioimageio/spec/model/v0_5.py +++ b/bioimageio/spec/model/v0_5.py @@ -180,11 +180,15 @@ class TensorId(LowerCaseIdentifier): - root_model = RootModel[Annotated[LowerCaseIdentifierAnno, MaxLen(32)]] + root_model: ClassVar[Type[RootModel[Any]]] = RootModel[ + Annotated[LowerCaseIdentifierAnno, MaxLen(32)] + ] class AxisId(LowerCaseIdentifier): - root_model = RootModel[Annotated[LowerCaseIdentifierAnno, MaxLen(16)]] + root_model: ClassVar[Type[RootModel[Any]]] = RootModel[ + Annotated[LowerCaseIdentifierAnno, MaxLen(16)] + ] NonBatchAxisId = Annotated[AxisId, Predicate(lambda x: x != "batch")] diff --git a/tests/test_internal/test_validated_string.py b/tests/test_internal/test_validated_string.py index 8be29eaee..12f81fdb1 100644 --- a/tests/test_internal/test_validated_string.py +++ b/tests/test_internal/test_validated_string.py @@ -1,3 +1,5 @@ +from typing import Any, ClassVar, Type + import annotated_types import pytest from pydantic import RootModel, ValidationError @@ -8,7 +10,9 @@ def test_valid_validated_string(): from bioimageio.spec._internal.validated_string import ValidatedString class V(ValidatedString): - root_model = RootModel[Annotated[str, annotated_types.Predicate(str.islower)]] + root_model: ClassVar[Type[RootModel[Any]]] = RootModel[ + Annotated[str, annotated_types.Predicate(str.islower)] + ] out = V("abc") assert isinstance(out, str) From 11b996afa730d65a3948b2f8d975d9451c661d40 Mon Sep 17 00:00:00 2001 From: fynnbe Date: Thu, 21 Mar 2024 00:16:14 +0100 Subject: [PATCH 25/36] fix _ValidatedString.validated --- bioimageio/spec/_internal/validated_string.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bioimageio/spec/_internal/validated_string.py b/bioimageio/spec/_internal/validated_string.py index cb0666e43..519cd66ca 100644 --- a/bioimageio/spec/_internal/validated_string.py +++ b/bioimageio/spec/_internal/validated_string.py @@ -16,7 +16,7 @@ class ValidatedString(str): def __new__(cls, object: object): self = super().__new__(cls, object) - self._validated = cls.root_model.model_validate(self) + self._validated = cls.root_model.model_validate(str(self)).root return self @classmethod From 33461525b3cec494a82d721a28370f27b48a56fb Mon Sep 17 00:00:00 2001 From: fynnbe Date: Thu, 21 Mar 2024 00:47:30 +0100 Subject: [PATCH 26/36] fix test_specific_reexports_generics --- tests/test_specific_reexports_generics.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/test_specific_reexports_generics.py b/tests/test_specific_reexports_generics.py index ae9665d69..6eaf35a54 100644 --- a/tests/test_specific_reexports_generics.py +++ b/tests/test_specific_reexports_generics.py @@ -15,6 +15,7 @@ "as_warning", "assert_never", "BioimageioYamlContent", + "ClassVar", "collections", "convert_from_older_format", "Converter", @@ -55,6 +56,7 @@ "settings", "string", "TAG_CATEGORIES", + "Type", "TypeVar", "Union", "V_suffix", From 40f36a423aef0ce09b230e5a3cc3e7fe54298e1d Mon Sep 17 00:00:00 2001 From: fynnbe Date: Thu, 21 Mar 2024 10:00:50 +0100 Subject: [PATCH 27/36] avoid pydantic warning --- tests/test_model/test_v0_4.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_model/test_v0_4.py b/tests/test_model/test_v0_4.py index acf5e7e40..590bda59a 100644 --- a/tests/test_model/test_v0_4.py +++ b/tests/test_model/test_v0_4.py @@ -329,7 +329,7 @@ def model_data(): description="Input 1", data_type="float32", axes="xyc", - shape=(128, 128, 3), + shape=[128, 128, 3], ), ], outputs=[ @@ -338,7 +338,7 @@ def model_data(): description="Output 1", data_type="float32", axes="xyc", - shape=(128, 128, 3), + shape=[128, 128, 3], ), ], name="Model", From 830547c61afc5b9748633b3e4f69b24edb4fee8f Mon Sep 17 00:00:00 2001 From: fynnbe Date: Thu, 21 Mar 2024 14:32:41 +0100 Subject: [PATCH 28/36] refactor internal bioimageio.yaml file name helpers --- bioimageio/spec/_internal/io.py | 57 +++++++++++++++++++------ bioimageio/spec/_internal/io_basics.py | 4 +- bioimageio/spec/_internal/io_utils.py | 35 ++------------- bioimageio/spec/_package.py | 8 ++-- bioimageio/spec/utils.py | 2 +- tests/test_internal/test_file_source.py | 8 ++-- tests/test_internal/test_types.py | 8 +--- 7 files changed, 63 insertions(+), 59 deletions(-) diff --git a/bioimageio/spec/_internal/io.py b/bioimageio/spec/_internal/io.py index 847c7c17c..3c1726449 100644 --- a/bioimageio/spec/_internal/io.py +++ b/bioimageio/spec/_internal/io.py @@ -13,6 +13,7 @@ ClassVar, Dict, Generic, + Iterable, List, Optional, Sequence, @@ -23,6 +24,7 @@ Union, ) from urllib.parse import urlparse, urlsplit, urlunsplit +from zipfile import ZipFile, is_zipfile import pooch import pydantic @@ -53,6 +55,7 @@ from .._internal._settings import settings from .._internal.io_basics import ( ALL_BIOIMAGEIO_YAML_NAMES, + ALTERNATIVE_BIOIMAGEIO_YAML_NAMES, BIOIMAGEIO_YAML, AbsoluteDirectory, AbsoluteFilePath, @@ -303,7 +306,7 @@ def validate(self, value: FileSource) -> FileSource: def wo_special_file_name(src: FileSource) -> FileSource: - if has_valid_rdf_name(src): + if has_valid_bioimageio_yaml_name(src): raise ValueError( f"'{src}' not allowed here as its filename is reserved to identify" + f" '{BIOIMAGEIO_YAML}' (or equivalent) files." @@ -397,20 +400,50 @@ def _package(value: FileSource, info: SerializationInfo) -> Union[str, Path, Fil ] -def has_valid_rdf_name(src: FileSource) -> bool: - return is_valid_rdf_name(extract_file_name(src)) +def has_valid_bioimageio_yaml_name(src: FileSource) -> bool: + return is_valid_bioimageio_yaml_name(extract_file_name(src)) -def is_valid_rdf_name(file_name: FileName) -> bool: - for special in ALL_BIOIMAGEIO_YAML_NAMES: - if file_name.endswith(special): +def is_valid_bioimageio_yaml_name(file_name: FileName) -> bool: + for bioimageio_name in ALL_BIOIMAGEIO_YAML_NAMES: + if file_name == bioimageio_name or file_name.endswith("." + bioimageio_name): return True return False -def ensure_has_valid_rdf_name(src: FileSource) -> FileSource: - if not has_valid_rdf_name(src): +def identify_bioimageio_yaml_file_name(file_names: Iterable[FileName]) -> FileName: + file_names = sorted(file_names) + for bioimageio_name in ALL_BIOIMAGEIO_YAML_NAMES: + for file_name in file_names: + if file_name == bioimageio_name or file_name.endswith( + "." + bioimageio_name + ): + return file_name + + raise ValueError( + f"No {BIOIMAGEIO_YAML} found in {file_names}. (Looking for '{BIOIMAGEIO_YAML}'" + + " or or any of the alterntive file names:" + + f" {ALTERNATIVE_BIOIMAGEIO_YAML_NAMES}, or any file with an extension of" + + f" those, e.g. 'anything.{BIOIMAGEIO_YAML}')." + ) + + +def find_bioimageio_yaml_file_name(path: Path) -> FileName: + if path.is_file(): + if not is_zipfile(path): + return path.name + + with ZipFile(path, "r") as f: + file_names = identify_bioimageio_yaml_file_name(f.namelist()) + else: + file_names = [p.name for p in path.glob("*")] + + return identify_bioimageio_yaml_file_name(file_names) + + +def ensure_has_valid_bioimageio_yaml_name(src: FileSource) -> FileSource: + if not has_valid_bioimageio_yaml_name(src): raise ValueError( f"'{src}' does not have a valid filename to identify" + f" '{BIOIMAGEIO_YAML}' (or equivalent) files." @@ -419,8 +452,8 @@ def ensure_has_valid_rdf_name(src: FileSource) -> FileSource: return src -def ensure_is_valid_rdf_name(file_name: FileName) -> FileName: - if not is_valid_rdf_name(file_name): +def ensure_is_valid_bioimageio_yaml_name(file_name: FileName) -> FileName: + if not is_valid_bioimageio_yaml_name(file_name): raise ValueError( f"'{file_name}' is not a valid filename to identify" + f" '{BIOIMAGEIO_YAML}' (or equivalent) files." @@ -448,14 +481,14 @@ def ensure_is_valid_rdf_name(file_name: FileName) -> FileName: class OpenedBioimageioYaml: content: BioimageioYamlContent original_root: Union[AbsoluteDirectory, RootHttpUrl] - original_file_name: str + original_file_name: FileName @dataclass class DownloadedFile: path: FilePath original_root: Union[AbsoluteDirectory, RootHttpUrl] - original_file_name: str + original_file_name: FileName class HashKwargs(TypedDict): diff --git a/bioimageio/spec/_internal/io_basics.py b/bioimageio/spec/_internal/io_basics.py index a2410298c..0c7c98169 100644 --- a/bioimageio/spec/_internal/io_basics.py +++ b/bioimageio/spec/_internal/io_basics.py @@ -8,6 +8,6 @@ AbsoluteDirectory = Annotated[DirectoryPath, Predicate(Path.is_absolute)] AbsoluteFilePath = Annotated[FilePath, Predicate(Path.is_absolute)] -BIOIMAGEIO_YAML = "rdf.yaml" -ALTERNATIVE_BIOIMAGEIO_YAML_NAMES = ("bioimageio.yaml", "model.yaml") +BIOIMAGEIO_YAML = "bioimageio.yaml" +ALTERNATIVE_BIOIMAGEIO_YAML_NAMES = ("rdf.yaml", "model.yaml") ALL_BIOIMAGEIO_YAML_NAMES = (BIOIMAGEIO_YAML,) + ALTERNATIVE_BIOIMAGEIO_YAML_NAMES diff --git a/bioimageio/spec/_internal/io_utils.py b/bioimageio/spec/_internal/io_utils.py index 51a63e54d..1f85c4f48 100644 --- a/bioimageio/spec/_internal/io_utils.py +++ b/bioimageio/spec/_internal/io_utils.py @@ -2,7 +2,7 @@ import warnings from contextlib import nullcontext from pathlib import Path -from typing import IO, Any, Dict, Iterable, Mapping, Optional, TextIO, Union, cast +from typing import IO, Any, Dict, Mapping, Optional, TextIO, Union, cast from zipfile import ZipFile, is_zipfile import numpy @@ -11,8 +11,9 @@ from ruyaml import YAML from typing_extensions import Unpack +from bioimageio.spec._internal.io import find_bioimageio_yaml_file_name + from .io import ( - ALL_BIOIMAGEIO_YAML_NAMES, BIOIMAGEIO_YAML, BioimageioYamlContent, FileDescr, @@ -21,7 +22,7 @@ YamlValue, download, ) -from .io_basics import ALTERNATIVE_BIOIMAGEIO_YAML_NAMES, FileName +from .io_basics import FileName from .types import FileSource, PermissiveFileSource yaml = YAML(typ="safe") @@ -84,34 +85,6 @@ def open_bioimageio_yaml( return OpenedBioimageioYaml(content, root, downloaded.original_file_name) -def identify_bioimageio_yaml_file_name(file_names: Iterable[FileName]) -> FileName: - file_names = sorted(file_names) - for bioimageio_name in ALL_BIOIMAGEIO_YAML_NAMES: - for fname in file_names: - if fname == bioimageio_name or fname.endswith(f".{bioimageio_name}"): - return fname - - raise ValueError( - f"No {BIOIMAGEIO_YAML} found in {file_names}. (Looking for '{BIOIMAGEIO_YAML}'" - + " or or any of the alterntive file names:" - + f" {ALTERNATIVE_BIOIMAGEIO_YAML_NAMES}, or any file with an extension of" - + f" those, e.g. 'anything.{BIOIMAGEIO_YAML}')." - ) - - -def find_bioimageio_yaml_file_name(path: Path) -> FileName: - if path.is_file(): - if not is_zipfile(path): - return path.name - - with ZipFile(path, "r") as f: - file_names = identify_bioimageio_yaml_file_name(f.namelist()) - else: - file_names = [p.name for p in path.glob("*")] - - return identify_bioimageio_yaml_file_name(file_names) - - def unzip( zip_file: Union[FilePath, ZipFile], out_path: Optional[DirectoryPath] = None, diff --git a/bioimageio/spec/_package.py b/bioimageio/spec/_package.py index 910a8debf..c044a2b1f 100644 --- a/bioimageio/spec/_package.py +++ b/bioimageio/spec/_package.py @@ -1,7 +1,7 @@ import collections.abc -from io import BytesIO import re import shutil +from io import BytesIO from pathlib import Path from tempfile import NamedTemporaryFile, mkdtemp from typing import IO, Dict, Literal, Optional, Sequence, Union, cast @@ -16,7 +16,7 @@ BioimageioYamlSource, YamlValue, download, - ensure_is_valid_rdf_name, + ensure_is_valid_bioimageio_yaml_name, ) from ._internal.io_basics import BIOIMAGEIO_YAML, AbsoluteFilePath, FileName from ._internal.io_utils import open_bioimageio_yaml, write_yaml, write_zip @@ -53,7 +53,9 @@ def get_resource_package_content( name=os_friendly_name, type=rd.type ) - bioimageio_yaml_file_name = ensure_is_valid_rdf_name(bioimageio_yaml_file_name) + bioimageio_yaml_file_name = ensure_is_valid_bioimageio_yaml_name( + bioimageio_yaml_file_name + ) content: Dict[FileName, Union[HttpUrl, AbsoluteFilePath]] = {} with PackagingContext( bioimageio_yaml_file_name=bioimageio_yaml_file_name, file_sources=content diff --git a/bioimageio/spec/utils.py b/bioimageio/spec/utils.py index 4cb57dd8a..d3ccbe855 100644 --- a/bioimageio/spec/utils.py +++ b/bioimageio/spec/utils.py @@ -1,5 +1,5 @@ from ._internal.io import download as download -from ._internal.io_utils import ( +from ._internal.io import ( identify_bioimageio_yaml_file_name as identify_bioimageio_yaml_file_name, ) from ._internal.io_utils import load_array as load_array diff --git a/tests/test_internal/test_file_source.py b/tests/test_internal/test_file_source.py index f82cb2b81..38eeaa2bc 100644 --- a/tests/test_internal/test_file_source.py +++ b/tests/test_internal/test_file_source.py @@ -15,13 +15,13 @@ ], ) def test_is_valid_rdf_name(name: FileName): - from bioimageio.spec._internal.io import is_valid_rdf_name + from bioimageio.spec._internal.io import is_valid_bioimageio_yaml_name - assert is_valid_rdf_name(name), name + assert is_valid_bioimageio_yaml_name(name), name @pytest.mark.parametrize("name", ["bioimageio.yml", "RDF.yaml", "smth.yaml"]) def test_is_invalid_rdf_name(name: FileName): - from bioimageio.spec._internal.io import is_valid_rdf_name + from bioimageio.spec._internal.io import is_valid_bioimageio_yaml_name - assert not is_valid_rdf_name(name), name + assert not is_valid_bioimageio_yaml_name(name), name diff --git a/tests/test_internal/test_types.py b/tests/test_internal/test_types.py index b886d560b..73f0ab773 100644 --- a/tests/test_internal/test_types.py +++ b/tests/test_internal/test_types.py @@ -1,4 +1,3 @@ -import typing from datetime import datetime from pathlib import Path @@ -70,7 +69,7 @@ ], ) def test_type_is_usable(name: str): - """check if a type can be instantiated or is a common Python type (e.g. Union or Literal)""" + """check if a type can be instantiated""" typ = getattr(types, name) if typ in TYPE_ARGS: args = TYPE_ARGS[typ] @@ -80,10 +79,7 @@ def test_type_is_usable(name: str): elif isinstance(typ, str): pass # ignore string constants else: - origin = typing.get_origin(typ) - assert origin in (dict, list, typing.Union, typing.Literal) or type(typ) in ( - typing.TypeVar, - ), name + raise ValueError(f"No idea how to test {name} -> {typ}") @pytest.mark.parametrize("path", [Path(__file__), Path()]) From e4cd9762693fe5083042c3497a004198589598b8 Mon Sep 17 00:00:00 2001 From: fynnbe Date: Thu, 21 Mar 2024 14:36:07 +0100 Subject: [PATCH 29/36] expose is_valid_bioimageio_yaml_name --- bioimageio/spec/utils.py | 1 + 1 file changed, 1 insertion(+) diff --git a/bioimageio/spec/utils.py b/bioimageio/spec/utils.py index d3ccbe855..3f13bf38b 100644 --- a/bioimageio/spec/utils.py +++ b/bioimageio/spec/utils.py @@ -2,5 +2,6 @@ from ._internal.io import ( identify_bioimageio_yaml_file_name as identify_bioimageio_yaml_file_name, ) +from ._internal.io import is_valid_bioimageio_yaml_name as is_valid_bioimageio_yaml_name from ._internal.io_utils import load_array as load_array from ._internal.io_utils import save_array as save_array From 7a52fe82a6c6d3129ebad708221c8f360eecf96a Mon Sep 17 00:00:00 2001 From: fynnbe Date: Thu, 21 Mar 2024 14:56:59 +0100 Subject: [PATCH 30/36] include local badges in package use InPackageIfLocalFileSource in badges.i.icon --- bioimageio/spec/_internal/io.py | 8 ++++++++ bioimageio/spec/generic/v0_2.py | 5 +++-- bioimageio/spec/generic/v0_3.py | 2 +- 3 files changed, 12 insertions(+), 3 deletions(-) diff --git a/bioimageio/spec/_internal/io.py b/bioimageio/spec/_internal/io.py index 3c1726449..86890d0be 100644 --- a/bioimageio/spec/_internal/io.py +++ b/bioimageio/spec/_internal/io.py @@ -398,6 +398,14 @@ def _package(value: FileSource, info: SerializationInfo) -> Union[str, Path, Fil AfterValidator(wo_special_file_name), include_in_package_serializer, ] +InPackageIfLocalFileSource = Union[ + Annotated[ + Union[FilePath, RelativeFilePath], + AfterValidator(wo_special_file_name), + include_in_package_serializer, + ], + Union[HttpUrl, pydantic.HttpUrl], +] def has_valid_bioimageio_yaml_name(src: FileSource) -> bool: diff --git a/bioimageio/spec/generic/v0_2.py b/bioimageio/spec/generic/v0_2.py index b302d3bb0..9d09083f2 100644 --- a/bioimageio/spec/generic/v0_2.py +++ b/bioimageio/spec/generic/v0_2.py @@ -31,6 +31,7 @@ from .._internal.field_warning import as_warning, issue_warning, warn from .._internal.io import ( BioimageioYamlContent, + InPackageIfLocalFileSource, WithSuffix, YamlValue, include_in_package_serializer, @@ -140,7 +141,7 @@ class BadgeDescr(Node, title="Custom badge"): """badge label to display on hover""" icon: Annotated[ - Union[HttpUrl, None], + Optional[InPackageIfLocalFileSource], Field(examples=["https://colab.research.google.com/assets/colab-badge.svg"]), ] = None """badge icon""" @@ -299,7 +300,7 @@ def _warn_empty_cite(cls, value: Any): """A URL to the Git repository where the resource is being developed.""" icon: Union[ - ImportantFileSource, Annotated[str, Len(min_length=1, max_length=2)], None + Annotated[str, Len(min_length=1, max_length=2)], ImportantFileSource, None ] = None """An icon for illustration""" diff --git a/bioimageio/spec/generic/v0_3.py b/bioimageio/spec/generic/v0_3.py index 58a9a6de0..47e2b5c8a 100644 --- a/bioimageio/spec/generic/v0_3.py +++ b/bioimageio/spec/generic/v0_3.py @@ -286,7 +286,7 @@ class GenericModelDescrBase(ResourceDescrBase): """A URL to the Git repository where the resource is being developed.""" icon: Union[ - ImportantFileSource, Annotated[str, Len(min_length=1, max_length=2)], None + Annotated[str, Len(min_length=1, max_length=2)], ImportantFileSource, None ] = None """An icon for illustration, e.g. on bioimage.io""" From 879c33d88c86cca91ea691d10994f7c05cd44a54 Mon Sep 17 00:00:00 2001 From: fynnbe Date: Thu, 21 Mar 2024 15:12:26 +0100 Subject: [PATCH 31/36] expose VALID_COVER_IMAGE_EXTENSIONS in subspecs --- bioimageio/spec/application/v0_2.py | 1 + bioimageio/spec/application/v0_3.py | 1 + bioimageio/spec/collection/v0_2.py | 1 + bioimageio/spec/collection/v0_3.py | 1 + bioimageio/spec/dataset/v0_2.py | 1 + bioimageio/spec/dataset/v0_3.py | 1 + bioimageio/spec/notebook/v0_2.py | 1 + bioimageio/spec/notebook/v0_3.py | 1 + tests/test_specific_reexports_generics.py | 2 +- 9 files changed, 9 insertions(+), 1 deletion(-) diff --git a/bioimageio/spec/application/v0_2.py b/bioimageio/spec/application/v0_2.py index a0e9348c8..fae82093b 100644 --- a/bioimageio/spec/application/v0_2.py +++ b/bioimageio/spec/application/v0_2.py @@ -7,6 +7,7 @@ from .._internal.io_basics import AbsoluteFilePath as AbsoluteFilePath from .._internal.types import ImportantFileSource from .._internal.url import HttpUrl as HttpUrl +from ..generic.v0_2 import VALID_COVER_IMAGE_EXTENSIONS as VALID_COVER_IMAGE_EXTENSIONS from ..generic.v0_2 import AttachmentsDescr as AttachmentsDescr from ..generic.v0_2 import Author as Author from ..generic.v0_2 import BadgeDescr as BadgeDescr diff --git a/bioimageio/spec/application/v0_3.py b/bioimageio/spec/application/v0_3.py index 31c39b7b2..d6192e6be 100644 --- a/bioimageio/spec/application/v0_3.py +++ b/bioimageio/spec/application/v0_3.py @@ -9,6 +9,7 @@ from .._internal.io_basics import AbsoluteFilePath as AbsoluteFilePath from .._internal.types import ImportantFileSource from .._internal.url import HttpUrl as HttpUrl +from ..generic.v0_3 import VALID_COVER_IMAGE_EXTENSIONS as VALID_COVER_IMAGE_EXTENSIONS from ..generic.v0_3 import Author as Author from ..generic.v0_3 import BadgeDescr as BadgeDescr from ..generic.v0_3 import CiteEntry as CiteEntry diff --git a/bioimageio/spec/collection/v0_2.py b/bioimageio/spec/collection/v0_2.py index 3a4d10588..f2cbfb5f6 100644 --- a/bioimageio/spec/collection/v0_2.py +++ b/bioimageio/spec/collection/v0_2.py @@ -22,6 +22,7 @@ from ..application import ApplicationDescr_v0_2, ApplicationDescr_v0_3 from ..dataset import DatasetDescr_v0_2, DatasetDescr_v0_3 from ..generic import GenericDescr_v0_2, GenericDescr_v0_3 +from ..generic.v0_2 import VALID_COVER_IMAGE_EXTENSIONS as VALID_COVER_IMAGE_EXTENSIONS from ..generic.v0_2 import AttachmentsDescr as AttachmentsDescr from ..generic.v0_2 import Author as Author from ..generic.v0_2 import BadgeDescr as BadgeDescr diff --git a/bioimageio/spec/collection/v0_3.py b/bioimageio/spec/collection/v0_3.py index f011a9713..a054c1943 100644 --- a/bioimageio/spec/collection/v0_3.py +++ b/bioimageio/spec/collection/v0_3.py @@ -32,6 +32,7 @@ from ..application import ApplicationDescr_v0_2, ApplicationDescr_v0_3 from ..dataset import DatasetDescr_v0_2, DatasetDescr_v0_3 from ..generic import GenericDescr_v0_2, GenericDescr_v0_3 +from ..generic.v0_3 import VALID_COVER_IMAGE_EXTENSIONS as VALID_COVER_IMAGE_EXTENSIONS from ..generic.v0_3 import Author as Author from ..generic.v0_3 import BadgeDescr as BadgeDescr from ..generic.v0_3 import CiteEntry as CiteEntry diff --git a/bioimageio/spec/dataset/v0_2.py b/bioimageio/spec/dataset/v0_2.py index ca1a3ad73..13336ed59 100644 --- a/bioimageio/spec/dataset/v0_2.py +++ b/bioimageio/spec/dataset/v0_2.py @@ -3,6 +3,7 @@ from .._internal.common_nodes import Node from .._internal.io_basics import AbsoluteFilePath as AbsoluteFilePath from .._internal.url import HttpUrl as HttpUrl +from ..generic.v0_2 import VALID_COVER_IMAGE_EXTENSIONS as VALID_COVER_IMAGE_EXTENSIONS from ..generic.v0_2 import AttachmentsDescr as AttachmentsDescr from ..generic.v0_2 import Author as Author from ..generic.v0_2 import BadgeDescr as BadgeDescr diff --git a/bioimageio/spec/dataset/v0_3.py b/bioimageio/spec/dataset/v0_3.py index 39c183a4e..cd0fa13d3 100644 --- a/bioimageio/spec/dataset/v0_3.py +++ b/bioimageio/spec/dataset/v0_3.py @@ -7,6 +7,7 @@ from .._internal.io import Sha256 as Sha256 from .._internal.io_basics import AbsoluteFilePath as AbsoluteFilePath from .._internal.url import HttpUrl as HttpUrl +from ..generic.v0_3 import VALID_COVER_IMAGE_EXTENSIONS as VALID_COVER_IMAGE_EXTENSIONS from ..generic.v0_3 import Author as Author from ..generic.v0_3 import BadgeDescr as BadgeDescr from ..generic.v0_3 import CiteEntry as CiteEntry diff --git a/bioimageio/spec/notebook/v0_2.py b/bioimageio/spec/notebook/v0_2.py index 9414c912d..a607a68cb 100644 --- a/bioimageio/spec/notebook/v0_2.py +++ b/bioimageio/spec/notebook/v0_2.py @@ -6,6 +6,7 @@ from .._internal.io import WithSuffix from .._internal.io_basics import AbsoluteFilePath as AbsoluteFilePath from .._internal.url import HttpUrl +from ..generic.v0_2 import VALID_COVER_IMAGE_EXTENSIONS as VALID_COVER_IMAGE_EXTENSIONS from ..generic.v0_2 import AttachmentsDescr as AttachmentsDescr from ..generic.v0_2 import Author as Author from ..generic.v0_2 import BadgeDescr as BadgeDescr diff --git a/bioimageio/spec/notebook/v0_3.py b/bioimageio/spec/notebook/v0_3.py index 020b70e63..ddefe1523 100644 --- a/bioimageio/spec/notebook/v0_3.py +++ b/bioimageio/spec/notebook/v0_3.py @@ -5,6 +5,7 @@ from .._internal.io import Sha256 as Sha256 from .._internal.io_basics import AbsoluteFilePath as AbsoluteFilePath from .._internal.url import HttpUrl as HttpUrl +from ..generic.v0_3 import VALID_COVER_IMAGE_EXTENSIONS as VALID_COVER_IMAGE_EXTENSIONS from ..generic.v0_3 import Author as Author from ..generic.v0_3 import BadgeDescr as BadgeDescr from ..generic.v0_3 import CiteEntry as CiteEntry diff --git a/tests/test_specific_reexports_generics.py b/tests/test_specific_reexports_generics.py index 6eaf35a54..35086190b 100644 --- a/tests/test_specific_reexports_generics.py +++ b/tests/test_specific_reexports_generics.py @@ -32,6 +32,7 @@ "ImportantFileSource", "include_in_package_serializer", "INFO", + "InPackageIfLocalFileSource", "issue_warning", "Len", "LicenseId", @@ -90,7 +91,6 @@ def get_members(m: ModuleType): "KNOWN_SPECIFIC_RESOURCE_TYPES", "ResourceDescrBase", "ResourceDescrType", - "VALID_COVER_IMAGE_EXTENSIONS", } GENERIC_v0_2_MEMBERS = { From 9690db3f7f33a29973b349c843a46e218a18452d Mon Sep 17 00:00:00 2001 From: fynnbe Date: Thu, 21 Mar 2024 15:14:26 +0100 Subject: [PATCH 32/36] fix test_specific_reexports_generics --- tests/test_specific_reexports_generics.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/test_specific_reexports_generics.py b/tests/test_specific_reexports_generics.py index 35086190b..099c315c2 100644 --- a/tests/test_specific_reexports_generics.py +++ b/tests/test_specific_reexports_generics.py @@ -16,6 +16,7 @@ "assert_never", "BioimageioYamlContent", "ClassVar", + "ClassVar", "collections", "convert_from_older_format", "Converter", @@ -27,6 +28,7 @@ "field_validator", "Field", "FileSource", + "FileSource", "Ge", "get_args", "ImportantFileSource", @@ -48,10 +50,12 @@ "NotEmpty", "Optional", "partial", + "PermissiveFileSource", "Predicate", "requests", "RestrictCharacters", "RootModel", + "S", "Self", "Sequence", "settings", From d48430bd844d0f0bb8af264c79e0efa77405d38a Mon Sep 17 00:00:00 2001 From: fynnbe Date: Thu, 21 Mar 2024 15:17:22 +0100 Subject: [PATCH 33/36] update license examples (also debugging ci fails non-existing test_license_id_in_model) --- bioimageio/spec/generic/v0_2.py | 2 +- bioimageio/spec/generic/v0_3.py | 2 +- bioimageio/spec/model/v0_4.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/bioimageio/spec/generic/v0_2.py b/bioimageio/spec/generic/v0_2.py index 9d09083f2..a26e85a57 100644 --- a/bioimageio/spec/generic/v0_2.py +++ b/bioimageio/spec/generic/v0_2.py @@ -396,7 +396,7 @@ def _convert_from_older_format( license: Annotated[ Optional[Union[LicenseId, DeprecatedLicenseId, str]], - Field(examples=["CC-BY-4.0", "MIT", "BSD-2-Clause"]), + Field(examples=["CC0-1.0", "MIT", "BSD-2-Clause"]), ] = None """A [SPDX license identifier](https://spdx.org/licenses/). We do not support custom license beyond the SPDX license list, if you need that please diff --git a/bioimageio/spec/generic/v0_3.py b/bioimageio/spec/generic/v0_3.py index 47e2b5c8a..1451dbdc8 100644 --- a/bioimageio/spec/generic/v0_3.py +++ b/bioimageio/spec/generic/v0_3.py @@ -235,7 +235,7 @@ class GenericModelDescrBase(ResourceDescrBase): LicenseId, "{value} is deprecated, see https://spdx.org/licenses/{value}.html", ), - Field(examples=["CC-BY-4.0", "MIT", "BSD-2-Clause"]), + Field(examples=["CC0-1.0", "MIT", "BSD-2-Clause"]), ] """A [SPDX license identifier](https://spdx.org/licenses/). We do not support custom license beyond the SPDX license list, if you need that please diff --git a/bioimageio/spec/model/v0_4.py b/bioimageio/spec/model/v0_4.py index e1316ab68..408c86759 100644 --- a/bioimageio/spec/model/v0_4.py +++ b/bioimageio/spec/model/v0_4.py @@ -900,7 +900,7 @@ class ModelDescr(GenericModelDescrBase, title="bioimage.io model specification") license: Annotated[ Union[LicenseId, str], warn(LicenseId, "Unknown license id '{value}'."), - Field(examples=["CC-BY-4.0", "MIT", "BSD-2-Clause"]), + Field(examples=["CC0-1.0", "MIT", "BSD-2-Clause"]), ] """A [SPDX license identifier](https://spdx.org/licenses/). We do notsupport custom license beyond the SPDX license list, if you need that please From eb4af738ef7a6b04c2855cd9e3eadbcbf3c1d296 Mon Sep 17 00:00:00 2001 From: fynnbe Date: Thu, 21 Mar 2024 15:20:07 +0100 Subject: [PATCH 34/36] fix test_types --- tests/test_internal/test_types.py | 5 +++++ tests/test_specific_reexports_generics.py | 1 + 2 files changed, 6 insertions(+) diff --git a/tests/test_internal/test_types.py b/tests/test_internal/test_types.py index 73f0ab773..fe18b3fbe 100644 --- a/tests/test_internal/test_types.py +++ b/tests/test_internal/test_types.py @@ -40,18 +40,23 @@ "annotations", "Any", "BeforeValidator", + "ClassVar", "datetime", "field_validation", + "FileSource", "FormatVersionPlaceholder", # a literal "ImportantFileSource", # an annoated union "iskeyword", "isoparse", "Literal", "NotEmpty", + "PermissiveFileSource", "PlainSerializer", "RootModel", + "S", "Sequence", "StringConstraints", + "Type", "TypeVar", "typing", "Union", diff --git a/tests/test_specific_reexports_generics.py b/tests/test_specific_reexports_generics.py index 099c315c2..a40c2fc65 100644 --- a/tests/test_specific_reexports_generics.py +++ b/tests/test_specific_reexports_generics.py @@ -62,6 +62,7 @@ "string", "TAG_CATEGORIES", "Type", + "Type", "TypeVar", "Union", "V_suffix", From 7e6a6213e8f077ed357d8d21f47dd7a9218a329e Mon Sep 17 00:00:00 2001 From: fynnbe Date: Thu, 21 Mar 2024 15:32:08 +0100 Subject: [PATCH 35/36] remove cyclic imports --- bioimageio/spec/_internal/field_validation.py | 8 ++++---- bioimageio/spec/_internal/io_utils.py | 3 +-- bioimageio/spec/generic/v0_3.py | 5 ++--- bioimageio/spec/model/v0_4.py | 1 + bioimageio/spec/model/v0_5.py | 8 +++++--- 5 files changed, 13 insertions(+), 12 deletions(-) diff --git a/bioimageio/spec/_internal/field_validation.py b/bioimageio/spec/_internal/field_validation.py index 03956162a..b0298da90 100644 --- a/bioimageio/spec/_internal/field_validation.py +++ b/bioimageio/spec/_internal/field_validation.py @@ -12,10 +12,10 @@ import requests -from bioimageio.spec._internal._settings import settings -from bioimageio.spec._internal.constants import KNOWN_GH_USERS, KNOWN_INVALID_GH_USERS -from bioimageio.spec._internal.field_warning import issue_warning -from bioimageio.spec._internal.validation_context import validation_context_var +from ._settings import settings +from .constants import KNOWN_GH_USERS, KNOWN_INVALID_GH_USERS +from .field_warning import issue_warning +from .validation_context import validation_context_var def is_valid_yaml_leaf_value(value: Any) -> bool: diff --git a/bioimageio/spec/_internal/io_utils.py b/bioimageio/spec/_internal/io_utils.py index 1f85c4f48..50afa5ecf 100644 --- a/bioimageio/spec/_internal/io_utils.py +++ b/bioimageio/spec/_internal/io_utils.py @@ -11,8 +11,6 @@ from ruyaml import YAML from typing_extensions import Unpack -from bioimageio.spec._internal.io import find_bioimageio_yaml_file_name - from .io import ( BIOIMAGEIO_YAML, BioimageioYamlContent, @@ -21,6 +19,7 @@ OpenedBioimageioYaml, YamlValue, download, + find_bioimageio_yaml_file_name, ) from .io_basics import FileName from .types import FileSource, PermissiveFileSource diff --git a/bioimageio/spec/generic/v0_3.py b/bioimageio/spec/generic/v0_3.py index eeeb918b4..855701ace 100644 --- a/bioimageio/spec/generic/v0_3.py +++ b/bioimageio/spec/generic/v0_3.py @@ -20,9 +20,6 @@ from pydantic import Field, RootModel, ValidationInfo, field_validator, model_validator from typing_extensions import Annotated -from bioimageio.spec._internal.field_validation import validate_gh_user -from bioimageio.spec._internal.validated_string import ValidatedString - from .._internal.common_nodes import ( Converter, Node, @@ -31,6 +28,7 @@ from .._internal.constants import ( TAG_CATEGORIES, ) +from .._internal.field_validation import validate_gh_user from .._internal.field_warning import as_warning, warn from .._internal.io import ( BioimageioYamlContent, @@ -50,6 +48,7 @@ ) from .._internal.types import RelativeFilePath as RelativeFilePath from .._internal.url import HttpUrl as HttpUrl +from .._internal.validated_string import ValidatedString from .._internal.validator_annotations import ( AfterValidator, Predicate, diff --git a/bioimageio/spec/model/v0_4.py b/bioimageio/spec/model/v0_4.py index 408c86759..724e60f25 100644 --- a/bioimageio/spec/model/v0_4.py +++ b/bioimageio/spec/model/v0_4.py @@ -54,6 +54,7 @@ from .._internal.validator_annotations import AfterValidator, RestrictCharacters from .._internal.version_type import Version as Version from .._internal.warning_levels import ALERT, INFO +from ..dataset.v0_2 import VALID_COVER_IMAGE_EXTENSIONS as VALID_COVER_IMAGE_EXTENSIONS from ..dataset.v0_2 import DatasetDescr as DatasetDescr from ..dataset.v0_2 import LinkedDataset as LinkedDataset from ..generic.v0_2 import AttachmentsDescr as AttachmentsDescr diff --git a/bioimageio/spec/model/v0_5.py b/bioimageio/spec/model/v0_5.py index 51235dcc9..6171e3274 100644 --- a/bioimageio/spec/model/v0_5.py +++ b/bioimageio/spec/model/v0_5.py @@ -45,9 +45,6 @@ ) from typing_extensions import Annotated, LiteralString, Self, assert_never -from bioimageio.spec._internal.validator_annotations import RestrictCharacters -from bioimageio.spec.generic.v0_3 import ResourceId as ResourceId - from .._internal.common_nodes import ( Converter, InvalidDescr, @@ -75,11 +72,15 @@ from .._internal.types import NotEmpty as NotEmpty from .._internal.url import HttpUrl as HttpUrl from .._internal.validation_context import validation_context_var +from .._internal.validator_annotations import RestrictCharacters from .._internal.version_type import Version as Version from .._internal.warning_levels import INFO from ..dataset.v0_3 import DatasetDescr as DatasetDescr from ..dataset.v0_3 import LinkedDataset as LinkedDataset from ..dataset.v0_3 import Uploader as Uploader +from ..generic.v0_3 import ( + VALID_COVER_IMAGE_EXTENSIONS as VALID_COVER_IMAGE_EXTENSIONS, +) from ..generic.v0_3 import Author as Author from ..generic.v0_3 import BadgeDescr as BadgeDescr from ..generic.v0_3 import CiteEntry as CiteEntry @@ -94,6 +95,7 @@ from ..generic.v0_3 import Maintainer as Maintainer from ..generic.v0_3 import OrcidId as OrcidId from ..generic.v0_3 import RelativeFilePath as RelativeFilePath +from ..generic.v0_3 import ResourceId as ResourceId from .v0_4 import Author as _Author_v0_4 from .v0_4 import BinarizeDescr as _BinarizeDescr_v0_4 from .v0_4 import CallableFromDepencency as CallableFromDepencency From 1e299397ba5b22f74a5b3e20b7ad6a4136aa5c9f Mon Sep 17 00:00:00 2001 From: fynnbe Date: Thu, 21 Mar 2024 15:32:55 +0100 Subject: [PATCH 36/36] fix test_license_id_in_model --- tests/test_internal/test_license_id.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/test_internal/test_license_id.py b/tests/test_internal/test_license_id.py index f917a237a..a3cee8b25 100644 --- a/tests/test_internal/test_license_id.py +++ b/tests/test_internal/test_license_id.py @@ -24,7 +24,6 @@ class Model(BaseModel): out = Model.model_validate({"lid": "CC-BY-4.0"}).lid assert isinstance(out, LicenseId) - assert not isinstance(out, str) def test_deprecated_license_id():