diff --git a/README.md b/README.md index 97b75bb6a..68c70923f 100644 --- a/README.md +++ b/README.md @@ -105,9 +105,13 @@ Made with [contrib.rocks](https://contrib.rocks). ### bioimageio.spec Python package -#### bioimageio.spec 0.5.2post6 (to be released) +#### bioimageio.spec 0.5.3 +* remove collection description * update SPDX license list +* update generic description to 0.3.1 +* update model description to 0.5.3 +* add timeout argument to all requests.get calls #### bioimageio.spec 0.5.2post5 @@ -283,6 +287,11 @@ Made with [contrib.rocks](https://contrib.rocks). ### Resource Description Format Versions +#### general 0.3.1 and model 0.5.3 + +* Non-breaking changes + * remove `version_number` in favor of using `version` + #### model 0.5.2 * Non-breaking changes diff --git a/bioimageio/spec/VERSION b/bioimageio/spec/VERSION index d2c266761..5e0138be3 100644 --- a/bioimageio/spec/VERSION +++ b/bioimageio/spec/VERSION @@ -1,3 +1,3 @@ { - "version": "0.5.2post5" + "version": "0.5.3" } diff --git a/bioimageio/spec/__init__.py b/bioimageio/spec/__init__.py index ba476e780..eac5514ba 100644 --- a/bioimageio/spec/__init__.py +++ b/bioimageio/spec/__init__.py @@ -3,7 +3,6 @@ """ from . import application as application -from . import collection as collection from . import dataset as dataset from . import generic as generic from . import model as model @@ -31,8 +30,6 @@ ) from .application import AnyApplicationDescr as AnyApplicationDescr from .application import ApplicationDescr as ApplicationDescr -from .collection import AnyCollectionDescr as AnyCollectionDescr -from .collection import CollectionDescr as CollectionDescr from .dataset import AnyDatasetDescr as AnyDatasetDescr from .dataset import DatasetDescr as DatasetDescr from .generic import AnyGenericDescr as AnyGenericDescr diff --git a/bioimageio/spec/_description.py b/bioimageio/spec/_description.py index e6b620926..a4a43ad21 100644 --- a/bioimageio/spec/_description.py +++ b/bioimageio/spec/_description.py @@ -13,9 +13,6 @@ from .application import AnyApplicationDescr, ApplicationDescr from .application.v0_2 import ApplicationDescr as ApplicationDescr02 from .application.v0_3 import ApplicationDescr as ApplicationDescr03 -from .collection import AnyCollectionDescr, CollectionDescr -from .collection.v0_2 import CollectionDescr as CollectionDescr02 -from .collection.v0_3 import CollectionDescr as CollectionDescr03 from .dataset import AnyDatasetDescr, DatasetDescr from .dataset.v0_2 import DatasetDescr as DatasetDescr02 from .dataset.v0_3 import DatasetDescr as DatasetDescr03 @@ -38,7 +35,6 @@ Annotated[ Union[ ApplicationDescr, - CollectionDescr, DatasetDescr, ModelDescr, NotebookDescr, @@ -53,7 +49,6 @@ SpecificResourceDescr = Annotated[ Union[ AnyApplicationDescr, - AnyCollectionDescr, AnyDatasetDescr, AnyModelDescr, AnyNotebookDescr, @@ -99,13 +94,6 @@ def dump_description( None: ApplicationDescr, } ), - "collection": MappingProxyType( - { - "0.2": CollectionDescr02, - "0.3": CollectionDescr03, - None: CollectionDescr, - } - ), "dataset": MappingProxyType( { "0.2": DatasetDescr02, diff --git a/bioimageio/spec/_internal/_settings.py b/bioimageio/spec/_internal/_settings.py index 3dd916f93..5216fe0ea 100644 --- a/bioimageio/spec/_internal/_settings.py +++ b/bioimageio/spec/_internal/_settings.py @@ -23,18 +23,19 @@ class Settings(BaseSettings, extra="ignore"): """url to bioimageio collection.json to resolve collection specific resource IDs. """ - collection_staged: str = ( - "https://uk1s3.embassy.ebi.ac.uk/public-datasets/bioimage.io/collection_staged.json" + collection_draft: str = ( + "https://uk1s3.embassy.ebi.ac.uk/public-datasets/bioimage.io/collection_draft.json" ) - """url to bioimageio collection_staged.json to resolve collection specific, staged - resource IDs.""" - - resolve_staged: bool = True - """Flag to resolve staged resource versions following the pattern - /staged/. - Note that anyone may stage a new resource version and that such a staged version - may not have been reviewed. - Set this flag to False to avoid this potential security risk.""" + """url to bioimageio collection_draft.json to resolve collection specific draft + versions of resources ending with '/draft'.""" + + resolve_draft: bool = True + """Flag to resolve draft resource versions following the pattern + /draft. + Note that anyone may stage a new draft and that such a draft version + may not have been reviewed yet. + Set this flag to False to avoid this potential security risk + and disallow loading draft versions.""" perform_io_checks: bool = True """wether or not to perform validation that requires file io, diff --git a/bioimageio/spec/_internal/io_utils.py b/bioimageio/spec/_internal/io_utils.py index a43c505e7..dba88c8a2 100644 --- a/bioimageio/spec/_internal/io_utils.py +++ b/bioimageio/spec/_internal/io_utils.py @@ -96,13 +96,13 @@ def open_bioimageio_yaml( collection = get_collection() if source not in collection: - if "/staged/" in source: - if settings.resolve_staged: - collection_url = settings.collection_staged + if isinstance(source, str) and source.endswith("/draft"): + if settings.resolve_draft: + collection_url = settings.collection_draft else: collection_url = "" logger.error( - "Did not try to resolve '{}' as BIOIMAGEIO_RESOLVE_STAGED is set to False", + "Did not try to resolve '{}' as BIOIMAGEIO_RESOLVE_DRAFT is set to False", source, ) else: @@ -115,7 +115,7 @@ def open_bioimageio_yaml( logger.info( "{} loading {} {} from {}", entry.emoji, - entry.id, + f"{entry.id}/{entry.version}", entry.version, entry.url, ) @@ -142,6 +142,7 @@ class _CollectionEntry(NamedTuple): url: str sha256: Optional[Sha256] version: str + doi: Optional[str] def _get_one_collection(url: str): @@ -160,81 +161,49 @@ def _get_one_collection(url: str): logger.error("`collection` field of {} has type {}", url, type(collection)) return ret - for entry in collection: - if entry["entry_sha256"] is None: - logger.debug("skipping {} with entry_sha256=None", entry["id"]) - continue - - if not isinstance(entry, dict): - logger.error("entry has type {}", type(entry)) - continue - if not isinstance(entry["id"], str): - logger.error("entry['id'] has type {}", type(entry["id"])) - continue - if not isinstance(entry["id_emoji"], str): - logger.error( - "{}.id_emoji has type {}", entry["id"], type(entry["id_emoji"]) + for raw_entry in collection: + try: + for i, (v, d) in enumerate(zip(raw_entry["versions"], raw_entry["dois"])): + entry = _CollectionEntry( + id=raw_entry["id"], + emoji=raw_entry.get("id_emoji", raw_entry.get("nickname_icon", "")), + url=raw_entry["rdf_source"], + sha256=raw_entry["rdf_sha256"], + version=v, + doi=d, + ) + ret[f"{raw_entry['id']}/{v}"] = entry + if i == 0: + # latest version + ret[raw_entry["id"]] = entry + if (concept_doi := raw_entry.get("concept_doi")) is not None: + ret[concept_doi] = entry + + if (nickname := raw_entry.get("nickname")) is not None: + ret[nickname] = entry + + if d is not None: + ret[d] = entry + + except Exception as e: + entry_id = ( + raw_entry.get("id", "unknown") + if isinstance(raw_entry, dict) + else "unknown" ) - continue - if not isinstance(entry["entry_source"], str): - logger.error( - "{}.entry_source has type {}", entry["id"], type(entry["entry_source"]) - ) - continue - if not isinstance(entry["entry_sha256"], str): logger.error( - "{}.entry_sha256 has type {}", entry["id"], type(entry["entry_sha256"]) + "failed to parse collection entry with `id={}`: {}", entry_id, e ) continue - c_entry = _CollectionEntry( - entry["id"], - entry["id_emoji"], - entry["entry_source"], - ( - None - if entry.get("entry_sha256") is None - else Sha256(entry["entry_sha256"]) - ), - version=str(entry["version_number"]), - ) - # set version specific entry - ret[c_entry.id + "/" + str(entry["version_number"])] = c_entry - - # set doi entry - doi = entry.get("doi") - if doi is not None: - ret[doi] = c_entry - - # update 'latest version' entry - if c_entry.id not in ret: - update = True - else: - old_v = ret[c_entry.id].version - v = c_entry.version - - if old_v.startswith("staged"): - update = not v.startswith("staged") or int( - v.replace("staged/", "") - ) > int(old_v.replace("staged/", "")) - else: - update = not v.startswith("staged") and int(v) > int(old_v) - - if update: - ret[c_entry.id] = c_entry - # set concept doi entry - concept_doi = entry.get("concept_doi") - if concept_doi is not None: - ret[concept_doi] = c_entry - return ret @lru_cache def get_collection() -> Mapping[str, _CollectionEntry]: try: - if settings.resolve_staged: - ret = _get_one_collection(settings.collection_staged) + if settings.resolve_draft: + ret = _get_one_collection(settings.collection_draft) else: ret = {} diff --git a/bioimageio/spec/application/v0_3.py b/bioimageio/spec/application/v0_3.py index aaff785ad..2969656ec 100644 --- a/bioimageio/spec/application/v0_3.py +++ b/bioimageio/spec/application/v0_3.py @@ -3,7 +3,6 @@ from pydantic import Field from typing_extensions import Annotated -from .._internal.common_nodes import Node from .._internal.io import FileDescr as FileDescr from .._internal.io_basics import AbsoluteFilePath as AbsoluteFilePath from .._internal.io_basics import Sha256 as Sha256 @@ -14,7 +13,7 @@ from ..generic.v0_3 import BadgeDescr as BadgeDescr from ..generic.v0_3 import CiteEntry as CiteEntry from ..generic.v0_3 import Doi as Doi -from ..generic.v0_3 import GenericDescrBase, ResourceId +from ..generic.v0_3 import GenericDescrBase, LinkedResourceNode, ResourceId from ..generic.v0_3 import LinkedResource as LinkedResource from ..generic.v0_3 import Maintainer as Maintainer from ..generic.v0_3 import OrcidId as OrcidId @@ -45,11 +44,8 @@ class ApplicationDescr(GenericDescrBase, title="bioimage.io application specific """The primary source of the application""" -class LinkedApplication(Node): +class LinkedApplication(LinkedResourceNode): """Reference to a bioimage.io application.""" id: ApplicationId """A valid application `id` from the bioimage.io collection.""" - - version_number: int - """version number (n-th published version, not the semantic version) of linked application""" diff --git a/bioimageio/spec/collection/__init__.py b/bioimageio/spec/collection/__init__.py deleted file mode 100644 index fc0a9b142..000000000 --- a/bioimageio/spec/collection/__init__.py +++ /dev/null @@ -1,20 +0,0 @@ -# autogen: start -""" -implementaions of all released minor versions are available in submodules: -- collection v0_2: `bioimageio.spec.collection.v0_2.CollectionDescr` -- collection v0_3: `bioimageio.spec.collection.v0_3.CollectionDescr` -""" -from typing import Union - -from pydantic import Discriminator -from typing_extensions import Annotated - -from .v0_2 import CollectionDescr as CollectionDescr_v0_2 -from .v0_3 import CollectionDescr as CollectionDescr -from .v0_3 import CollectionDescr as CollectionDescr_v0_3 - -AnyCollectionDescr = Annotated[ - Union[CollectionDescr_v0_2, CollectionDescr_v0_3], Discriminator("format_version") -] -"""Union of any released collection desription""" -# autogen: stop diff --git a/bioimageio/spec/collection/v0_2.py b/bioimageio/spec/collection/v0_2.py deleted file mode 100644 index 8391ae5c4..000000000 --- a/bioimageio/spec/collection/v0_2.py +++ /dev/null @@ -1,295 +0,0 @@ -import collections.abc -from functools import partial -from types import MappingProxyType -from typing import Any, Dict, List, Literal, Optional, Union, get_args - -from pydantic import ( - PrivateAttr, - model_validator, -) -from typing_extensions import Self - -from .._build_description import build_description_impl, get_rd_class_impl -from .._internal.common_nodes import InvalidDescr, Node -from .._internal.field_warning import issue_warning -from .._internal.io import BioimageioYamlContent, YamlValue -from .._internal.io_basics import AbsoluteFilePath as AbsoluteFilePath -from .._internal.io_utils import open_bioimageio_yaml -from .._internal.types import NotEmpty -from .._internal.url import HttpUrl as HttpUrl -from .._internal.validation_context import validation_context_var -from .._internal.warning_levels import ALERT -from ..application import ApplicationDescr_v0_2, ApplicationDescr_v0_3 -from ..dataset import DatasetDescr_v0_2, DatasetDescr_v0_3 -from ..generic import GenericDescr_v0_2, GenericDescr_v0_3 -from ..generic.v0_2 import VALID_COVER_IMAGE_EXTENSIONS as VALID_COVER_IMAGE_EXTENSIONS -from ..generic.v0_2 import AttachmentsDescr as AttachmentsDescr -from ..generic.v0_2 import Author as Author -from ..generic.v0_2 import BadgeDescr as BadgeDescr -from ..generic.v0_2 import CiteEntry as CiteEntry -from ..generic.v0_2 import Doi as Doi -from ..generic.v0_2 import FileSource, GenericDescrBase -from ..generic.v0_2 import LinkedResource as LinkedResource -from ..generic.v0_2 import Maintainer as Maintainer -from ..generic.v0_2 import OrcidId as OrcidId -from ..generic.v0_2 import RelativeFilePath as RelativeFilePath -from ..generic.v0_2 import ResourceId as ResourceId -from ..generic.v0_2 import Uploader as Uploader -from ..generic.v0_2 import Version as Version -from ..model import ModelDescr_v0_4, ModelDescr_v0_5 -from ..notebook import NotebookDescr_v0_2, NotebookDescr_v0_3 - - -class CollectionId(ResourceId): - pass - - -EntryDescr = Union[ - ApplicationDescr_v0_2, - DatasetDescr_v0_2, - GenericDescr_v0_2, - ModelDescr_v0_4, - NotebookDescr_v0_2, -] - -_ENTRY_DESCR_MAP = MappingProxyType( - { - None: MappingProxyType( - { - "0.2": GenericDescr_v0_2, - "0.3": GenericDescr_v0_3, - None: GenericDescr_v0_2, - } - ), - "generic": MappingProxyType( - { - "0.2": GenericDescr_v0_2, - "0.3": GenericDescr_v0_3, - None: GenericDescr_v0_2, - } - ), - "application": MappingProxyType( - { - "0.2": ApplicationDescr_v0_2, - "0.3": ApplicationDescr_v0_3, - None: ApplicationDescr_v0_2, - } - ), - "dataset": MappingProxyType( - { - "0.2": DatasetDescr_v0_2, - "0.3": DatasetDescr_v0_3, - None: DatasetDescr_v0_2, - } - ), - "notebook": MappingProxyType( - { - "0.2": NotebookDescr_v0_2, - "0.3": NotebookDescr_v0_3, - None: NotebookDescr_v0_2, - } - ), - "model": MappingProxyType( - { - "0.3": ModelDescr_v0_4, - "0.4": ModelDescr_v0_4, - "0.5": ModelDescr_v0_5, - None: ModelDescr_v0_4, - } - ), - } -) - - -class CollectionEntry(Node, extra="allow"): - """A valid resource description (RD). - The entry RD is based on the collection description itself. - Fields are added/overwritten by the content of `rdf_source` if `rdf_source` is specified, - and finally added/overwritten by any fields specified directly in the entry. - Except for the `id` field, fields are overwritten entirely, their content is not merged! - The final `id` for each collection entry is composed of the collection's `id` - and the entry's 'sub-'`id`, specified remotely as part of `rdf_source` or superseeded in-place, - such that the `final_entry_id = /`""" - - rdf_source: Optional[FileSource] = None - """resource description file (RDF) source to load entry from""" - - id: Optional[ResourceId] = None - """Collection entry sub id overwriting `rdf_source.id`. - The full collection entry's id is the collection's base id, followed by this sub id and separated by a slash '/'.""" - - _descr: Optional[EntryDescr] = PrivateAttr(None) - - @property - def rdf_update(self) -> Dict[str, YamlValue]: - return self.model_extra or {} - - @property - def descr(self) -> Optional[EntryDescr]: - if self._descr is None: - issue_warning( - "Collection entry description not set. Is this entry part of a" - + " Collection? A collection entry only has its `descr` set if it is part" - + " of a valid collection description.", - value=None, - severity=ALERT, - ) - - return self._descr - - -class CollectionDescr( - GenericDescrBase, extra="allow", title="bioimage.io collection specification" -): - """A bioimage.io collection describes several other bioimage.io resources. - Note that collections cannot be nested; resources listed under `collection` may not be collections themselves. - """ - - type: Literal["collection"] = "collection" - - id: Optional[CollectionId] = None - """Model zoo (bioimage.io) wide, unique identifier (assigned by bioimage.io)""" - - collection: NotEmpty[List[CollectionEntry]] - """Collection entries""" - - @model_validator(mode="after") - def finalize_entries(self) -> Self: - context = validation_context_var.get() - common_entry_content = { - k: v - for k, v in self.model_dump(mode="json", exclude_unset=True).items() - if k not in ("id", "collection") - } - common_badges = common_entry_content.pop( - "badges", None - ) # `badges` not valid for model entries - base_id: Optional[CollectionId] = self.id - - seen_entry_ids: Dict[str, int] = {} - - for i, entry in enumerate(self.collection): - entry_data: Dict[str, Any] = dict(common_entry_content) - # set entry specific root as it might be adapted in the presence of an external entry source - entry_root = context.root - entry_file_name = context.file_name - - if entry.rdf_source is not None: - if not context.perform_io_checks: - issue_warning( - "Skipping IO dependent validation (perform_io_checks=False)", - value=entry.rdf_source, - msg_context=dict(i=i), - field=f"collection[{i}]", - ) - continue - - external_data = open_bioimageio_yaml(entry.rdf_source) - # add/overwrite common collection entry content with external source - entry_data.update(external_data.content) - entry_root = external_data.original_root - entry_file_name = external_data.original_file_name - - # add/overwrite common+external entry content with in-place entry update - entry_data.update(entry.rdf_update) - - # also update explicitly specified `id` field data - if entry.id is not None: - entry_data["id"] = entry.id - - if "id" in entry_data: - entry_id = str(entry_data["id"]) - if (seen_i := seen_entry_ids.get(entry_id)) is not None: - raise ValueError( - f"Dublicate `id` '{entry_data['id']}' in" - + f" collection[{seen_i}]/collection[{i}]" - ) - - seen_entry_ids[entry_id] = i - else: - raise ValueError(f"Missing `id` for entry {i}") - - if base_id is not None: - entry_data["id"] = f"{base_id}/{entry_data['id']}" - - type_ = entry_data.get("type") - if type_ == "collection": - raise ValueError( - f"collection[{i}] has invalid entry type; collections may not be" - + " nested!" - ) - - if ( - type_ != "model" - and common_badges is not None - and "badges" not in entry_data - ): - # set badges from the collection root for non-model resources if not set for this specific entry - entry_data["badges"] = common_badges - - entry_descr = build_description_impl( - entry_data, - context=context.replace(root=entry_root, file_name=entry_file_name), - get_rd_class=partial( - get_rd_class_impl, descriptions_map=_ENTRY_DESCR_MAP - ), - ) - assert entry_descr.validation_summary is not None - if isinstance(entry_descr, InvalidDescr): - raise ValueError( - "Invalid collection entry" - + f" collection[{i}]:\n" - + f"{entry_descr.validation_summary.format(hide_source=True, hide_env=True, root_loc=('collection', i))}" - ) - elif isinstance( - entry_descr, get_args(EntryDescr) - ): # TODO: use EntryDescr as union (py>=3.10) - entry._descr = entry_descr # type: ignore - else: - raise ValueError( - f"{entry_descr.type} {entry_descr.format_version} entries" - + f" are not allowed in {self.type} {self.format_version}." - ) - - return self - - @model_validator(mode="before") - @classmethod - def move_groups_to_collection_field( - cls, data: BioimageioYamlContent - ) -> BioimageioYamlContent: - if data.get("format_version") not in ("0.2.0", "0.2.1"): - return data - - if "collection" in data and data["collection"] is not None: - if not isinstance(data["collection"], collections.abc.Sequence): - raise ValueError( - "Expected `collection` to not be present, or to be a list" - ) - - data["collection"] = list(data["collection"]) - else: - data["collection"] = [] - - for group in ["application", "model", "dataset", "notebook"]: - if group in data: - data["collection"] += data[group] # type: ignore - data["collection"][-1]["type"] = group - - config = data.get("config") - if config and isinstance(config, dict): - id_ = config.pop("id", data.get("id")) - if id_ is not None: - data["id"] = id_ - - return data - - -class LinkedCollection(Node): - """Reference to a bioimage.io collection.""" - - id: CollectionId - """A valid collection `id` from the bioimage.io collection.""" - - version_number: Optional[int] = None - """version number (n-th published version, not the semantic version) of linked collection""" diff --git a/bioimageio/spec/collection/v0_3.py b/bioimageio/spec/collection/v0_3.py deleted file mode 100644 index 25f836094..000000000 --- a/bioimageio/spec/collection/v0_3.py +++ /dev/null @@ -1,350 +0,0 @@ -from functools import partial -from types import MappingProxyType -from typing import ( - TYPE_CHECKING, - Any, - Dict, - List, - Literal, - Optional, - Union, - cast, - get_args, -) - -from pydantic import PrivateAttr, model_validator -from typing_extensions import Self - -from .._build_description import build_description_impl, get_rd_class_impl -from .._internal.common_nodes import InvalidDescr, Node -from .._internal.field_warning import issue_warning -from .._internal.io import FileDescr as FileDescr -from .._internal.io import YamlValue -from .._internal.io_basics import AbsoluteFilePath as AbsoluteFilePath -from .._internal.io_basics import Sha256 as Sha256 -from .._internal.io_utils import open_bioimageio_yaml -from .._internal.types import FileSource, NotEmpty -from .._internal.url import HttpUrl as HttpUrl -from .._internal.validation_context import ( - validation_context_var, -) -from .._internal.warning_levels import ALERT -from ..application import ApplicationDescr_v0_2, ApplicationDescr_v0_3 -from ..dataset import DatasetDescr_v0_2, DatasetDescr_v0_3 -from ..generic import GenericDescr_v0_2, GenericDescr_v0_3 -from ..generic.v0_3 import VALID_COVER_IMAGE_EXTENSIONS as VALID_COVER_IMAGE_EXTENSIONS -from ..generic.v0_3 import Author as Author -from ..generic.v0_3 import BadgeDescr as BadgeDescr -from ..generic.v0_3 import CiteEntry as CiteEntry -from ..generic.v0_3 import Doi as Doi -from ..generic.v0_3 import ( - GenericDescrBase, - ResourceId, - _author_conv, # pyright: ignore[reportPrivateUsage] - _maintainer_conv, # pyright: ignore[reportPrivateUsage] -) -from ..generic.v0_3 import LinkedResource as LinkedResource -from ..generic.v0_3 import Maintainer as Maintainer -from ..generic.v0_3 import OrcidId as OrcidId -from ..generic.v0_3 import RelativeFilePath as RelativeFilePath -from ..generic.v0_3 import Uploader as Uploader -from ..generic.v0_3 import Version as Version -from ..model import ModelDescr_v0_4, ModelDescr_v0_5 -from ..notebook import NotebookDescr_v0_2, NotebookDescr_v0_3 -from .v0_2 import CollectionDescr as _CollectionDescr_v0_2 - - -class CollectionId(ResourceId): - pass - - -EntryDescr = Union[ - ApplicationDescr_v0_2, - ApplicationDescr_v0_3, - DatasetDescr_v0_2, - DatasetDescr_v0_3, - ModelDescr_v0_4, - ModelDescr_v0_5, - NotebookDescr_v0_2, - NotebookDescr_v0_3, - GenericDescr_v0_2, - GenericDescr_v0_3, -] - -_ENTRY_DESCR_MAP = MappingProxyType( - { - None: MappingProxyType( - { - "0.2": GenericDescr_v0_2, - "0.3": GenericDescr_v0_3, - None: GenericDescr_v0_3, - } - ), - "generic": MappingProxyType( - { - "0.2": GenericDescr_v0_2, - "0.3": GenericDescr_v0_3, - None: GenericDescr_v0_3, - } - ), - "application": MappingProxyType( - { - "0.2": ApplicationDescr_v0_2, - "0.3": ApplicationDescr_v0_3, - None: ApplicationDescr_v0_3, - } - ), - "dataset": MappingProxyType( - { - "0.2": DatasetDescr_v0_2, - "0.3": DatasetDescr_v0_3, - None: DatasetDescr_v0_3, - } - ), - "notebook": MappingProxyType( - { - "0.2": NotebookDescr_v0_2, - "0.3": NotebookDescr_v0_3, - None: NotebookDescr_v0_3, - } - ), - "model": MappingProxyType( - { - "0.3": ModelDescr_v0_4, - "0.4": ModelDescr_v0_4, - "0.5": ModelDescr_v0_5, - None: ModelDescr_v0_5, - } - ), - } -) - - -class CollectionEntry(Node, extra="allow"): - """A collection entry description is based on the collection description itself. - Fields are added/overwritten by the content of `descr_source` if `descr_source` is set, - and finally added/overwritten by any fields specified directly in the entry. - Except for the `id` field, fields are overwritten entirely, their content is not merged! - The final `id` for each collection entry is composed of the collection's `id` - and the entry's 'sub-'`id`, specified externally in `descr_source` or superseeded in-place, - such that the `final_entry_id = /`""" - - entry_source: Optional[FileSource] = None - """an external source this entry description is based on""" - - entry_sha256: Optional[Sha256] = None - """SHA256 of `entry_source`""" - - id: Optional[ResourceId] = None - """Collection entry sub id overwriting `rdf_source.id`. - The full collection entry's id is the collection's base id, followed by this sub id and separated by a slash '/'.""" - - _descr: Optional[EntryDescr] = PrivateAttr(None) - - @property - def entry_update(self) -> Dict[str, YamlValue]: - return self.model_extra or {} - - @property - def descr(self) -> Optional[EntryDescr]: - if self._descr is None: - issue_warning( - "Collection entry description not set. Is this entry part of a" - + " Collection? A collection entry only has its `descr` set if it is part" - + " of a valid collection description.", - value=None, - severity=ALERT, - ) - - return self._descr - - -class CollectionDescr( - GenericDescrBase, extra="allow", title="bioimage.io collection specification" -): - """A bioimage.io collection resource description file (collection RDF) describes a collection of bioimage.io - resources. - The resources listed in a collection RDF have types other than 'collection'; collections cannot be nested. - """ - - type: Literal["collection"] = "collection" - - id: Optional[CollectionId] = None - """Model zoo (bioimage.io) wide, unique identifier (assigned by bioimage.io)""" - - parent: Optional[CollectionId] = None - """The description from which this one is derived""" - - collection: NotEmpty[List[CollectionEntry]] - """Collection entries""" - - @model_validator(mode="after") - def finalize_entries(self) -> Self: - context = validation_context_var.get() - common_entry_content = { - k: v - for k, v in self.model_dump(mode="json", exclude_unset=True).items() - if k not in ("id", "collection") - } - common_badges = common_entry_content.pop( - "badges", None - ) # `badges` not valid for model entries - base_id: Optional[CollectionId] = self.id - - seen_entry_ids: Dict[str, int] = {} - - for i, entry in enumerate(self.collection): - entry_data: Dict[str, Any] = dict(common_entry_content) - # set entry specific root as it might be adapted in the presence of an external entry source - entry_root = context.root - entry_file_name = context.file_name - - if entry.entry_source is None: - if entry.entry_sha256 is None: - raise ValueError( - f"Got unexpected `entry_sha256` {entry.entry_sha256} for unspecified `entry_source`" - ) - else: - if not context.perform_io_checks: - issue_warning( - "Skipping IO relying validation for collection[{i}]", - value=entry.entry_source, - msg_context=dict(i=i), - ) - continue - - external_data = open_bioimageio_yaml( - entry.entry_source, sha256=entry.entry_sha256 - ) - # add/overwrite common collection entry content with external source - entry_data.update(external_data.content) - entry_root = external_data.original_root - entry_file_name = external_data.original_file_name - - # add/overwrite common+external entry content with in-place entry update - entry_data.update(entry.entry_update) - - # also update explicitly specified `id` field data - if entry.id is not None: - entry_data["id"] = entry.id - - if "id" in entry_data: - entry_id = str(entry_data["id"]) - if (seen_i := seen_entry_ids.get(entry_id)) is not None: - raise ValueError( - f"Dublicate `id` '{entry_data['id']}' in" - + f" collection[{seen_i}]/collection[{i}]" - ) - - seen_entry_ids[entry_id] = i - else: - raise ValueError(f"Missing `id` for entry {i}") - - if base_id is not None: - entry_data["id"] = f"{base_id}/{entry_data['id']}" - - type_ = entry_data.get("type") - if type_ == "collection": - raise ValueError( - f"collection[{i}].type may not be 'collection'; collections may not" - + " be nested!" - ) - - if ( - type_ != "model" - and common_badges is not None - and "badges" not in entry_data - ): - # set badges from the collection root for non-model resources if not set for this specific entry - entry_data["badges"] = common_badges - - entry_descr = build_description_impl( - entry_data, - context=context.replace(root=entry_root, file_name=entry_file_name), - get_rd_class=partial( - get_rd_class_impl, descriptions_map=_ENTRY_DESCR_MAP - ), - ) - - assert entry_descr.validation_summary is not None - if isinstance(entry_descr, InvalidDescr): - raise ValueError( - "Invalid collection entry" - + f" collection[{i}]:\n" - + f"{entry_descr.validation_summary.format(hide_source=True, hide_env=True, root_loc=('collection', i))}" - ) - elif isinstance( - entry_descr, get_args(EntryDescr) - ): # TODO: use EntryDescr as union (py>=3.10) - entry._descr = entry_descr # type: ignore - else: - raise ValueError( - f"{entry_descr.type} {entry_descr.format_version} entries " - + f"are not allowed in {self.type} {self.format_version}." - ) - return self - - @model_validator(mode="before") - @classmethod - def _convert(cls, data: Dict[str, Any], /) -> Dict[str, Any]: - if ( - data.get("type") == "collection" - and isinstance(fv := data.get("format_version"), str) - and fv.startswith("0.2.") - ): - old = _CollectionDescr_v0_2.load(data) - if isinstance(old, InvalidDescr): - return data - - return cast( - Dict[str, Any], - (cls if TYPE_CHECKING else dict)( - attachments=( - [] - if old.attachments is None - else [FileDescr(source=f) for f in old.attachments.files] - ), - authors=[_author_conv.convert_as_dict(a) for a in old.authors], - badges=old.badges, - cite=[ - {"text": c.text, "doi": c.doi, "url": c.url} for c in old.cite - ], - collection=[ - (CollectionEntry if TYPE_CHECKING else dict)( - entry_source=entry.rdf_source, id=entry.id, **entry.rdf_update # type: ignore - ) - for entry in old.collection - ], - config=old.config, - covers=old.covers, - description=old.description, - documentation=old.documentation, - format_version="0.3.0", - git_repo=old.git_repo, - icon=old.icon, - id=old.id, - license=old.license, - links=old.links, - maintainers=[ - _maintainer_conv.convert_as_dict(m) for m in old.maintainers - ], - name=old.name, - tags=old.tags, - type=old.type, - uploader=old.uploader, - version=old.version, - **(old.model_extra or {}), - ), - ) - - return data - - -class LinkedCollection(Node): - """Reference to a bioimage.io collection.""" - - id: CollectionId - """A valid collection `id` from the bioimage.io collection.""" - - version_number: int - """version number (n-th published version, not the semantic version) of linked collection""" diff --git a/bioimageio/spec/dataset/v0_3.py b/bioimageio/spec/dataset/v0_3.py index 241d593d6..bbb8bf212 100644 --- a/bioimageio/spec/dataset/v0_3.py +++ b/bioimageio/spec/dataset/v0_3.py @@ -2,7 +2,7 @@ from pydantic import model_validator -from .._internal.common_nodes import InvalidDescr, Node +from .._internal.common_nodes import InvalidDescr from .._internal.io import FileDescr as FileDescr from .._internal.io_basics import AbsoluteFilePath as AbsoluteFilePath from .._internal.io_basics import Sha256 as Sha256 @@ -14,6 +14,7 @@ from ..generic.v0_3 import ( DocumentationSource, GenericDescrBase, + LinkedResourceNode, _author_conv, # pyright: ignore[reportPrivateUsage] _maintainer_conv, # pyright: ignore[reportPrivateUsage] ) @@ -101,11 +102,8 @@ def _convert(cls, data: Dict[str, Any], /) -> Dict[str, Any]: return data -class LinkedDataset(Node): +class LinkedDataset(LinkedResourceNode): """Reference to a bioimage.io dataset.""" id: DatasetId """A valid dataset `id` from the bioimage.io collection.""" - - version_number: int - """version number (n-th published version, not the semantic version) of linked dataset""" diff --git a/bioimageio/spec/generic/v0_3.py b/bioimageio/spec/generic/v0_3.py index 0fc9cbc26..fbffd43f4 100644 --- a/bioimageio/spec/generic/v0_3.py +++ b/bioimageio/spec/generic/v0_3.py @@ -40,9 +40,9 @@ from .._internal.io import FileDescr as FileDescr from .._internal.io_basics import AbsoluteFilePath from .._internal.io_basics import Sha256 as Sha256 -from .._internal.license_id import LicenseId +from .._internal.license_id import DeprecatedLicenseId as DeprecatedLicenseId +from .._internal.license_id import LicenseId as LicenseId from .._internal.types import ( - DeprecatedLicenseId, ImportantFileSource, NotEmpty, ) @@ -187,9 +187,6 @@ class LinkedResource(Node): id: ResourceId """A valid resource `id` from the official bioimage.io collection.""" - version_number: int - """version number (n-th published version, not the semantic version) of linked resource""" - class GenericModelDescrBase(ResourceDescrBase): """Base for all resource descriptions including of model descriptions""" @@ -364,8 +361,14 @@ def warn_about_tag_categories( version: Optional[Version] = None """The version of the resource following SemVer 2.0.""" - version_number: Optional[int] = None - """version number (n-th published version, not the semantic version)""" + @model_validator(mode="before") + def _remove_version_number(cls, value: Union[Any, Dict[Any, Any]]): + if isinstance(value, dict): + vn: Any = value.pop("version_number", None) + if vn is not None and "id" in value: + value["id"] = f"{value['id']}/{vn}" + + return value class GenericDescrBase(GenericModelDescrBase): @@ -433,3 +436,15 @@ def check_specific_types(cls, value: str) -> str: ) return value + + +class LinkedResourceNode(Node): + + @model_validator(mode="before") + def _remove_version_number(cls, value: Union[Any, Dict[Any, Any]]): + if isinstance(value, dict): + vn: Any = value.pop("version_number", None) + if vn is not None and "id" in value: + value["id"] = f"{value['id']}/{vn}" + + return value diff --git a/bioimageio/spec/model/v0_5.py b/bioimageio/spec/model/v0_5.py index 05da86f46..0687b6842 100644 --- a/bioimageio/spec/model/v0_5.py +++ b/bioimageio/spec/model/v0_5.py @@ -91,6 +91,7 @@ from ..generic.v0_3 import ( DocumentationSource, GenericModelDescrBase, + LinkedResourceNode, _author_conv, # pyright: ignore[reportPrivateUsage] _maintainer_conv, # pyright: ignore[reportPrivateUsage] ) @@ -1975,15 +1976,12 @@ class ModelId(ResourceId): pass -class LinkedModel(Node): +class LinkedModel(LinkedResourceNode): """Reference to a bioimage.io model.""" id: ModelId """A valid model `id` from the bioimage.io collection.""" - version_number: int - """version number (n-th published version, not the semantic version) of linked model""" - class _DataDepSize(NamedTuple): min: int @@ -2009,7 +2007,7 @@ class ModelDescr(GenericModelDescrBase, title="bioimage.io model specification") These fields are typically stored in a YAML file which we call a model resource description file (model RDF). """ - format_version: Literal["0.5.0"] = "0.5.0" + format_version: Literal["0.5.3"] = "0.5.3" """Version of the bioimage.io model description specification used. When creating a new model always use the latest micro/patch version described here. The `format_version` is important for any consumer software to understand how to parse the fields. @@ -2595,7 +2593,7 @@ def conv_authors(auths: Optional[Sequence[_Author_v0_4]]): covers=src.covers, description=src.description, documentation=src.documentation, # pyright: ignore[reportArgumentType] - format_version="0.5.0", + format_version="0.5.3", git_repo=src.git_repo, # pyright: ignore[reportArgumentType] icon=src.icon, id=None if src.id is None else ModelId(src.id), diff --git a/bioimageio/spec/notebook/v0_3.py b/bioimageio/spec/notebook/v0_3.py index b786c218d..f388e191e 100644 --- a/bioimageio/spec/notebook/v0_3.py +++ b/bioimageio/spec/notebook/v0_3.py @@ -1,6 +1,5 @@ from typing import Literal, Optional -from .._internal.common_nodes import Node from .._internal.io import FileDescr as FileDescr from .._internal.io_basics import AbsoluteFilePath as AbsoluteFilePath from .._internal.io_basics import Sha256 as Sha256 @@ -10,7 +9,7 @@ from ..generic.v0_3 import BadgeDescr as BadgeDescr from ..generic.v0_3 import CiteEntry as CiteEntry from ..generic.v0_3 import Doi as Doi -from ..generic.v0_3 import GenericDescrBase +from ..generic.v0_3 import GenericDescrBase, LinkedResourceNode from ..generic.v0_3 import LinkedResource as LinkedResource from ..generic.v0_3 import Maintainer as Maintainer from ..generic.v0_3 import OrcidId as OrcidId @@ -40,11 +39,8 @@ class NotebookDescr(GenericDescrBase, title="bioimage.io notebook specification" """The Jupyter notebook""" -class LinkedNotebook(Node): +class LinkedNotebook(LinkedResourceNode): """Reference to a bioimage.io notebook.""" id: NotebookId """A valid notebook `id` from the bioimage.io collection.""" - - version_number: int - """version number (n-th published version, not the semantic version) of linked notebook""" diff --git a/example_descriptions/collections/partner_collection/bioimageio.yaml b/example_descriptions/collections/partner_collection/bioimageio.yaml deleted file mode 100644 index 66bf03c72..000000000 --- a/example_descriptions/collections/partner_collection/bioimageio.yaml +++ /dev/null @@ -1,15 +0,0 @@ -format_version: 0.2.4 -type: collection -name: Partner Collection -description: "Resources for BioImgage.IO curated by the partner team." -authors: - - name: Fynn Beuttenmueller - github_user: fynnbe -cite: - - text: bioimage.io - url: "https://www.biorxiv.org/content/10.1101/2022.06.07.495102v1" -tags: [bioimage.io, partner-software] -id: partner -license: MIT -collection: - - rdf_source: datasets/dummy-dataset/dummy_entry.yaml diff --git a/example_descriptions/collections/partner_collection/datasets/dummy-dataset/README.md b/example_descriptions/collections/partner_collection/datasets/dummy-dataset/README.md deleted file mode 100644 index f5e06430e..000000000 --- a/example_descriptions/collections/partner_collection/datasets/dummy-dataset/README.md +++ /dev/null @@ -1 +0,0 @@ -# Amazing documentation diff --git a/example_descriptions/collections/partner_collection/datasets/dummy-dataset/dummy_entry.yaml b/example_descriptions/collections/partner_collection/datasets/dummy-dataset/dummy_entry.yaml deleted file mode 100644 index 4c324f9d7..000000000 --- a/example_descriptions/collections/partner_collection/datasets/dummy-dataset/dummy_entry.yaml +++ /dev/null @@ -1,8 +0,0 @@ -description: Dummy dataset for testing purposes only. -documentation: README.md -format_version: 0.2.4 -name: Dummy Data -tags: [dummy] -type: dataset -id: dummy-dataset -attachments: {} diff --git a/example_descriptions/collections/unet2d_nuclei_broad_coll/README.md b/example_descriptions/collections/unet2d_nuclei_broad_coll/README.md deleted file mode 100644 index 0d31864f6..000000000 --- a/example_descriptions/collections/unet2d_nuclei_broad_coll/README.md +++ /dev/null @@ -1,3 +0,0 @@ -# U-Net 2d Nuclei Broad Example - -A 2d U-Net trained on the nuclei broad dataset diff --git a/example_descriptions/collections/unet2d_nuclei_broad_coll/bioimageio.yaml b/example_descriptions/collections/unet2d_nuclei_broad_coll/bioimageio.yaml deleted file mode 100644 index 6c17e5630..000000000 --- a/example_descriptions/collections/unet2d_nuclei_broad_coll/bioimageio.yaml +++ /dev/null @@ -1,93 +0,0 @@ -format_version: 0.2.4 -type: collection - -name: UNet 2D Nuclei Broad - -description: A 2d U-Net trained on the nuclei broad dataset. -authors: - - name: "Constantin Pape;@bioimage-io" - affiliation: "EMBL Heidelberg" - orcid: "0000-0001-6562-7187" - - name: "Fynn Beuttenmueller" - affiliation: "EMBL Heidelberg" - orcid: "0000-0002-8567-6389" -maintainers: - - name: "Constantin Pape" - github_user: constantinpape - -cite: - - text: "Ronneberger, Olaf et al. U-net: Convolutional networks for biomedical image segmentation. MICCAI 2015." - doi: 10.1007/978-3-319-24574-4_28 - - text: "2018 Data Science Bowl" - url: https://www.kaggle.com/c/data-science-bowl-2018 - -git_repo: https://github.com/bioimage-io/spec-bioimage-io/tree/main/example_descriptions/models/unet2d_nuclei_broad -# tags: [unet2d, pytorch, nucleus, segmentation, dsb2018] # tags is optional for collection -license: MIT - -documentation: README.md -covers: [cover0.png] -timestamp: 2019-12-11T12:22:32Z - -inputs: - - name: raw - description: raw input - axes: bcyx - data_type: float32 - data_range: ["-inf", "inf"] - shape: [1, 1, 512, 512] - preprocessing: - - name: zero_mean_unit_variance - kwargs: - mode: per_sample - axes: yx - -outputs: - - name: probability - description: probability in [0,1] - axes: bcyx - data_type: float32 - data_range: ["-inf", "inf"] - halo: [0, 0, 32, 32] - shape: - reference_tensor: raw - scale: [1.0, 1.0, 1.0, 1.0] - offset: [0.0, 0.0, 0.0, 0.0] - -test_inputs: [test_input.npy] -test_outputs: [test_output.npy] - -sample_inputs: [test_input.npy] -sample_outputs: [test_output.npy] - -version: 1 - -collection: - # - id: with_rdf_source_url - # rdf_source: https://raw.githubusercontent.com/bioimage-io/spec-bioimage-io/main/example_descriptions/models/unet2d_nuclei_broad/rdf.yaml - # name: UNet 2D Nuclei Broad (latest) - - id: in_place_0.4.10 - format_version: 0.4.10 - type: model - weights: - pytorch_state_dict: - dependencies: conda:environment.yaml - authors: - - name: "Constantin Pape;@bioimage-io" - affiliation: "EMBL Heidelberg" - orcid: "0000-0001-6562-7187" - sha256: e4d3885bccbe41cbf6c1d825f3cd2b707c7021ead5593156007e407a16b27cf2 - source: https://zenodo.org/records/3446812/files/unet2d_weights.torch - architecture: unet2d.py:UNet2d - architecture_sha256: 7cdd8332dc3e3735e71c328f81b63a9ac86c028f80522312484ca9a4027d4ce1 - kwargs: { input_channels: 1, output_channels: 1 } - onnx: - sha256: f1f086d5e340f9d4d7001a1b62a2b835f9b87a2fb5452c4fe7d8cc821bdf539c - source: weights.onnx - opset_version: 12 - parent: pytorch_state_dict - torchscript: - sha256: 62fa1c39923bee7d58a192277e0dd58f2da9ee810662addadd0f44a3784d9210 - source: weights.pt - parent: pytorch_state_dict - attachments: {} diff --git a/example_descriptions/collections/unet2d_nuclei_broad_coll/cover0.png b/example_descriptions/collections/unet2d_nuclei_broad_coll/cover0.png deleted file mode 100644 index cac369cb1..000000000 Binary files a/example_descriptions/collections/unet2d_nuclei_broad_coll/cover0.png and /dev/null differ diff --git a/example_descriptions/collections/unet2d_nuclei_broad_coll/environment.yaml b/example_descriptions/collections/unet2d_nuclei_broad_coll/environment.yaml deleted file mode 100644 index 4297c6caa..000000000 --- a/example_descriptions/collections/unet2d_nuclei_broad_coll/environment.yaml +++ /dev/null @@ -1,8 +0,0 @@ -name: - unet2d_nuclei_broad -channels: - - conda-forge - - defaults -dependencies: - - pytorch - - numpy diff --git a/example_descriptions/collections/unet2d_nuclei_broad_coll/test_input.npy b/example_descriptions/collections/unet2d_nuclei_broad_coll/test_input.npy deleted file mode 100644 index 228057f8d..000000000 Binary files a/example_descriptions/collections/unet2d_nuclei_broad_coll/test_input.npy and /dev/null differ diff --git a/example_descriptions/collections/unet2d_nuclei_broad_coll/test_output.npy b/example_descriptions/collections/unet2d_nuclei_broad_coll/test_output.npy deleted file mode 100644 index de18244e2..000000000 Binary files a/example_descriptions/collections/unet2d_nuclei_broad_coll/test_output.npy and /dev/null differ diff --git a/example_descriptions/collections/unet2d_nuclei_broad_coll/unet2d.py b/example_descriptions/collections/unet2d_nuclei_broad_coll/unet2d.py deleted file mode 100644 index 7fba44dd5..000000000 --- a/example_descriptions/collections/unet2d_nuclei_broad_coll/unet2d.py +++ /dev/null @@ -1,86 +0,0 @@ -# type: ignore -import torch -import torch.nn as nn - - -class Upsample(nn.Module): - def __init__(self, scale_factor, mode="bilinear"): - super().__init__() - self.scale_factor = scale_factor - self.mode = mode - - def forward(self, input): - return nn.functional.interpolate( - input, scale_factor=self.scale_factor, mode=self.mode, align_corners=False - ) - - -class UNet2d(nn.Module): - def __init__(self, input_channels, output_channels, training=False): - super().__init__() - self.input_channels = input_channels - self.output_channels = output_channels - self.n_levels = 3 - - self.encoders = nn.ModuleList( - [ - self.conv_layer(self.input_channels, 16), - self.conv_layer(16, 32), - self.conv_layer(32, 64), - ] - ) - self.downsamplers = nn.ModuleList([self.downsampler()] * self.n_levels) - - self.base = self.conv_layer(64, 128) - - self.decoders = nn.ModuleList( - [self.conv_layer(128, 64), self.conv_layer(64, 32), self.conv_layer(32, 16)] - ) - self.upsamplers = nn.ModuleList( - [self.upsampler(128, 64), self.upsampler(64, 32), self.upsampler(32, 16)] - ) - - self.output = nn.Conv2d(16, self.output_channels, 1) - self.training = training - - def conv_layer(self, in_channels, out_channels): - kernel_size = 3 - padding = 1 - return nn.Sequential( - nn.Conv2d(in_channels, out_channels, kernel_size, padding=padding), - nn.Conv2d(out_channels, out_channels, kernel_size, padding=padding), - nn.ReLU(inplace=True), - ) - - def downsampler(self): - return nn.MaxPool2d(2) - - def upsampler(self, in_channels, out_channels): - return nn.Sequential(Upsample(2), nn.Conv2d(in_channels, out_channels, 1)) - - def forward(self, input): - x = input - - from_encoder = [] - for encoder, sampler in zip(self.encoders, self.downsamplers): - x = encoder(x) - from_encoder.append(x) - x = sampler(x) - - x = self.base(x) - - for decoder, sampler, enc in zip( - self.decoders, self.upsamplers, from_encoder[::-1] - ): - x = sampler(x) - x = torch.cat([enc, x], dim=1) - x = decoder(x) - - x = self.output(x) - - # apply a sigmoid directly if we are in inference mode - if not self.training: - # postprocessing - x = torch.sigmoid(x) - - return x diff --git a/example_descriptions/collections/unet2d_nuclei_broad_coll/weights.onnx b/example_descriptions/collections/unet2d_nuclei_broad_coll/weights.onnx deleted file mode 100644 index f1da5c884..000000000 Binary files a/example_descriptions/collections/unet2d_nuclei_broad_coll/weights.onnx and /dev/null differ diff --git a/example_descriptions/collections/unet2d_nuclei_broad_coll/weights.pt b/example_descriptions/collections/unet2d_nuclei_broad_coll/weights.pt deleted file mode 100644 index 834e73f24..000000000 Binary files a/example_descriptions/collections/unet2d_nuclei_broad_coll/weights.pt and /dev/null differ diff --git a/example_descriptions/models/unet2d_nuclei_broad/bioimageio.yaml b/example_descriptions/models/unet2d_nuclei_broad/bioimageio.yaml index f7d2ccb8a..91c514534 100644 --- a/example_descriptions/models/unet2d_nuclei_broad/bioimageio.yaml +++ b/example_descriptions/models/unet2d_nuclei_broad/bioimageio.yaml @@ -1,5 +1,5 @@ type: model -format_version: 0.5.0 +format_version: 0.5.3 name: UNet 2D Nuclei Broad description: A 2d U-Net trained on the nuclei broad dataset. @@ -122,5 +122,4 @@ weights: pytorch_version: "1.5.1" training_data: - id: ilastik/covid_if_training_data # note: not the real training data - version_number: 1 + id: ilastik/covid_if_training_data/1 # note: not the real training data diff --git a/scripts/generate_spec_documentation.py b/scripts/generate_spec_documentation.py index 194b58c9e..0bab7b24e 100644 --- a/scripts/generate_spec_documentation.py +++ b/scripts/generate_spec_documentation.py @@ -16,7 +16,6 @@ from bioimageio.spec import ( ResourceDescr, application, - collection, dataset, generic, model, @@ -512,7 +511,6 @@ def main(dist: Path): dist.mkdir(exist_ok=True, parents=True) export_module_documentations(dist, application) - export_module_documentations(dist, collection) export_module_documentations(dist, dataset) export_module_documentations(dist, generic) export_module_documentations(dist, model) diff --git a/scripts/generate_version_submodule_imports.py b/scripts/generate_version_submodule_imports.py index e11b7aa5c..bedd50108 100644 --- a/scripts/generate_version_submodule_imports.py +++ b/scripts/generate_version_submodule_imports.py @@ -42,7 +42,6 @@ def main(command: Literal["check", "generate"]): "generic", "model", "dataset", - "collection", "notebook", "application", ]: diff --git a/tests/test_io.py b/tests/test_io.py index ef40e7953..bba0db10d 100644 --- a/tests/test_io.py +++ b/tests/test_io.py @@ -22,8 +22,7 @@ def test_load_non_existing_rdf(): ], ) def test_load_by_id(rid: str): - from bioimageio.spec import InvalidDescr, load_description + from bioimageio.spec._internal.io_utils import open_bioimageio_yaml - model = load_description(rid) - assert not isinstance(model, InvalidDescr) - assert model.id == "invigorating-lab-coat" + rdf = open_bioimageio_yaml(rid).content + assert rdf["id"] == "invigorating-lab-coat" diff --git a/tests/test_model/test_format_version.py b/tests/test_model/test_format_version.py new file mode 100644 index 000000000..755e0b36a --- /dev/null +++ b/tests/test_model/test_format_version.py @@ -0,0 +1,9 @@ +def test_format_version_matches_library(): + from bioimageio.spec import __version__ + from bioimageio.spec.model import ModelDescr + + version_wo_post = __version__.split("post")[0] + assert ModelDescr.implemented_format_version == version_wo_post, ( + ModelDescr.implemented_format_version, + __version__, + ) diff --git a/tests/test_model/test_v0_5.py b/tests/test_model/test_v0_5.py index d26db49b4..569f64fb4 100644 --- a/tests/test_model/test_v0_5.py +++ b/tests/test_model/test_v0_5.py @@ -230,7 +230,7 @@ def model_data(): documentation=UNET2D_ROOT / "README.md", license=LicenseId("MIT"), git_repo=HttpUrl("https://github.com/bioimage-io/core-bioimage-io-python"), - format_version="0.5.0", + format_version="0.5.3", description="description", authors=[ Author(name="Author 1", affiliation="Affiliation 1"), @@ -454,7 +454,7 @@ def test_output_ref_shape_too_small(model_data: Dict[str, Any]): def test_model_has_parent_with_id(model_data: Dict[str, Any]): - model_data["parent"] = dict(id="10.5281/zenodo.5764892", version_number=1) + model_data["parent"] = dict(id="10.5281/zenodo.5764892/6647674") summary = validate_format( model_data, context=ValidationContext(perform_io_checks=False) ) diff --git a/tests/test_specific_reexports_generics.py b/tests/test_specific_reexports_generics.py index a40c2fc65..dddae506d 100644 --- a/tests/test_specific_reexports_generics.py +++ b/tests/test_specific_reexports_generics.py @@ -3,7 +3,7 @@ import pytest -from bioimageio.spec import application, collection, dataset, generic, model, notebook +from bioimageio.spec import application, dataset, generic, model, notebook IGNORE_MEMBERS = { "AfterValidator", @@ -94,6 +94,7 @@ def get_members(m: ModuleType): "GenericDescrBase", "GenericModelDescrBase", "KNOWN_SPECIFIC_RESOURCE_TYPES", + "LinkedResourceNode", "ResourceDescrBase", "ResourceDescrType", } @@ -110,12 +111,10 @@ def get_members(m: ModuleType): "generic_members,specific", [ (GENERIC_v0_2_MEMBERS, application.v0_2), - (GENERIC_v0_2_MEMBERS, collection.v0_2), (GENERIC_v0_2_MEMBERS, dataset.v0_2), (GENERIC_v0_2_MEMBERS, model.v0_4), (GENERIC_v0_2_MEMBERS, notebook.v0_2), (GENERIC_v0_3_MEMBERS, application.v0_3), - (GENERIC_v0_3_MEMBERS, collection.v0_3), (GENERIC_v0_3_MEMBERS, dataset.v0_3), (GENERIC_v0_3_MEMBERS, model.v0_5), (GENERIC_v0_3_MEMBERS, notebook.v0_3), diff --git a/tests/test_utils.py b/tests/test_utils.py new file mode 100644 index 000000000..d1a8a3173 --- /dev/null +++ b/tests/test_utils.py @@ -0,0 +1,7 @@ +def test_license_zenodo(): + """test that all licenses are known or not known by zenodo. + Run scripts/update_spdx_licenses_zenodo.py to fix this test""" + from bioimageio.spec.utils import get_spdx_licenses + + for lic in get_spdx_licenses()["licenses"]: + assert isinstance(lic["isKnownByZenodo"], bool), lic["licenseId"] diff --git a/tests/utils.py b/tests/utils.py index 15c91c9c5..6e3c1715d 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -33,7 +33,6 @@ from bioimageio.spec._internal.url import HttpUrl from bioimageio.spec._internal.validation_context import ValidationContext from bioimageio.spec.application.v0_2 import ApplicationDescr as ApplicationDescr02 -from bioimageio.spec.collection.v0_2 import CollectionDescr as CollectionDescr02 from bioimageio.spec.dataset.v0_2 import DatasetDescr as DatasetDescr02 from bioimageio.spec.generic._v0_2_converter import DOI_PREFIXES from bioimageio.spec.generic.v0_2 import GenericDescr as GenericDescr02 @@ -195,7 +194,6 @@ def check_bioimageio_yaml( ( ModelDescr04, ApplicationDescr02, - CollectionDescr02, DatasetDescr02, GenericDescr02, NotebookDescr02,