diff --git a/.gitignore b/.gitignore index d8d84865..e720787d 100644 --- a/.gitignore +++ b/.gitignore @@ -2,3 +2,6 @@ .*.ttl .git_submodule_init.done.log .lib.done.log +.venv.done.log +_* +venv diff --git a/CONTRIBUTE.md b/CONTRIBUTE.md index 4bacd610..f29b8fba 100644 --- a/CONTRIBUTE.md +++ b/CONTRIBUTE.md @@ -1,6 +1,17 @@ -# Contributing to the CASE ontology +# Contributing to the UCO ontology ## Testing prerelease states Practices for users interested in testing prerelease states are documented on the [Cyber Domain Ontology website](https://cyberdomainontology.org/ontology/development/#testing-prereleases). + + +## Using Protégé catalog files + +Interested users of `catalog-v001.xml` files, e.g. users of [Protégé](https://protege.stanford.edu/), can use these XML files to interact with UCO as local files. To do so, UCO must be `git-clone`'d with Git submodules also cloned. This can be done with the following commands: + +* `git clone --recursive https://github.com/ucoProject/UCO.git` (all users) +* `git clone https://github.com/ucoProject/UCO.git ; cd UCO ; make` (macOS or Linux users) + - The narrowest setup operation strictly for purposes of supporting the `catalog-v001.xml` files is to run `make .git_submodule_init.done.log` instead of the default `make all`. + +Protégé should not require network connectivity to load imported ontologies after the above commands are run. diff --git a/Makefile b/Makefile index 21c5d61f..de556a38 100644 --- a/Makefile +++ b/Makefile @@ -13,8 +13,11 @@ SHELL := /bin/bash +PYTHON3 ?= $(shell which python3) + all: \ - .lib.done.log + .lib.done.log \ + .venv.done.log $(MAKE) \ --directory ontology @@ -44,9 +47,31 @@ all: \ --directory lib touch $@ +# The two CASE-Utility... files are to trigger rebuilds based on command-line interface changes or version increments. +.venv.done.log: \ + dependencies/CASE-Utility-SHACL-Inheritance-Reviewer/case_shacl_inheritance_reviewer/__init__.py \ + dependencies/CASE-Utility-SHACL-Inheritance-Reviewer/setup.cfg \ + requirements.txt + rm -rf venv + $(PYTHON3) -m venv \ + venv + source venv/bin/activate \ + && pip install \ + --upgrade \ + pip \ + setuptools \ + wheel + source venv/bin/activate \ + && pip install \ + dependencies/CASE-Utility-SHACL-Inheritance-Reviewer + source venv/bin/activate \ + && pip install \ + --requirement requirements.txt + touch $@ + check: \ - .git_submodule_init.done.log \ - .lib.done.log + .lib.done.log \ + .venv.done.log $(MAKE) \ --directory ontology \ check @@ -59,7 +84,10 @@ clean: \ clean-ontology @rm -f \ .git_submodule_init.done.log \ - .lib.done.log + .lib.done.log \ + .venv.done.log + @rm -rf \ + venv clean-ontology: @$(MAKE) \ @@ -70,3 +98,15 @@ clean-tests: @$(MAKE) \ --directory tests \ clean + +# Maintain timestamp order. +dependencies/CASE-Utility-SHACL-Inheritance-Reviewer/case_shacl_inheritance_reviewer/__init__.py: \ + .git_submodule_init.done.log + touch -c $@ + test -r $@ + +# Maintain timestamp order. +dependencies/CASE-Utility-SHACL-Inheritance-Reviewer/setup.cfg: \ + .git_submodule_init.done.log + touch -c $@ + test -r $@ diff --git a/README.md b/README.md index ff1d015e..b52b0aae 100644 --- a/README.md +++ b/README.md @@ -13,8 +13,8 @@ Through this approach not only are domain-focused representations defined consis The purpose of this repository is to provide a foundation for broader community involvement in defining what to represent and how. ### Current Release -The current release of UCO is 1.1.0. +The current release of UCO is 1.2.0. -UCO 1.1.0 incorporates required refinements and updates, building on the stable 1.0.0 release. Following [SemVer](https://semver.org/spec/v2.0.0.html), additive improvements will continue to be accepted, but backwards-incompatible changes will be scheduled only for the 2.0.0 release, which will come after at least 6 months to possibly 12 months. +UCO 1.2.0 incorporates required refinements and updates, building on the stable 1.0.0 release. Following [SemVer](https://semver.org/spec/v2.0.0.html), additive improvements will continue to be accepted, but backwards-incompatible changes will be scheduled only for the 2.0.0 release, which will come after at least 6 months to possibly 12 months. -More detail of improvements is documented in the [UCO 1.1.0 release notes](https://unifiedcyberontology.org/releases/1.1.0/). +More detail of improvements is documented in the [UCO 1.2.0 release notes](https://unifiedcyberontology.org/releases/1.2.0/). diff --git a/etc/dependency_files.tsv b/etc/dependency_files.tsv new file mode 100644 index 00000000..876633b6 --- /dev/null +++ b/etc/dependency_files.tsv @@ -0,0 +1,2 @@ +http://purl.org/co/ ${top_srcdir}/dependencies/collections-ontology/collections.owl +http://purl.org/spar/error ${top_srcdir}/dependencies/error/docs/current/error.ttl diff --git a/etc/domain_directories.tsv b/etc/domain_directories.tsv new file mode 100644 index 00000000..10672584 --- /dev/null +++ b/etc/domain_directories.tsv @@ -0,0 +1 @@ +https://ontology.unifiedcyberontology.org/ ${top_srcdir}/ontology/ diff --git a/ontology/co/Makefile b/ontology/co/Makefile index 82b6e784..f4ba8dea 100644 --- a/ontology/co/Makefile +++ b/ontology/co/Makefile @@ -30,7 +30,8 @@ check_reference_basenames := $(foreach ttl_basename,$(ttl_basenames),.check-$(tt check_targets := $(foreach ttl_basename,$(ttl_basenames),check-$(ttl_basename)) all: \ - $(check_reference_basenames) + $(check_reference_basenames) \ + catalog-v001.xml .check-%.ttl: \ %.ttl \ @@ -43,8 +44,25 @@ all: \ --target-format turtle mv $@_ $@ +catalog-v001.xml: \ + $(top_srcdir)/.venv.done.log \ + $(top_srcdir)/etc/domain_directories.tsv \ + $(top_srcdir)/etc/dependency_files.tsv \ + $(top_srcdir)/src/create-catalog-v001.xml.py \ + $(ttl_basenames) + rm -f _$@ + source $(top_srcdir)/venv/bin/activate \ + && python3 $(top_srcdir)/src/create-catalog-v001.xml.py \ + _$@ \ + $(top_srcdir)/etc/domain_directories.tsv \ + $(top_srcdir)/etc/dependency_files.tsv \ + "$(top_srcdir)" \ + $(ttl_basenames) + mv _$@ $@ + check: \ - $(check_targets) + $(check_targets) \ + catalog-v001.xml # Reminder: diff exits non-0 on finding any differences. # Reminder: The $^ automatic Make variable is the name of all recipe prerequisites. @@ -55,5 +73,6 @@ check-%.ttl: \ || (echo "ERROR:ontology/co/Makefile:The local $< does not match the normalized version. If the above reported changes look fine, run 'cp .check-$< $<' while in the sub-folder ontology/co/ to get a file ready to commit to Git." >&2 ; exit 1) clean: - @rm -f $(check_reference_basenames) - + @rm -f \ + $(check_reference_basenames) \ + catalog-v001.xml diff --git a/ontology/co/catalog-v001.xml b/ontology/co/catalog-v001.xml new file mode 100644 index 00000000..43384aef --- /dev/null +++ b/ontology/co/catalog-v001.xml @@ -0,0 +1,6 @@ + + + + + + diff --git a/ontology/co/co.ttl b/ontology/co/co.ttl index 75239f47..97b18776 100644 --- a/ontology/co/co.ttl +++ b/ontology/co/co.ttl @@ -12,10 +12,10 @@ a owl:Ontology ; rdfs:label "uco-co"@en ; rdfs:comment "This ontology defines SHACL shapes to supplement the OWL 2 DL definitions in the Collections Ontology."@en ; - owl:backwardCompatibleWith uco-co:1.0.0 ; + owl:backwardCompatibleWith uco-co:1.1.0 ; owl:imports ; - owl:priorVersion uco-co:1.0.0 ; - owl:versionIRI uco-co:1.1.0 ; + owl:priorVersion uco-co:1.1.0 ; + owl:versionIRI uco-co:1.2.0 ; . co:ListItem diff --git a/ontology/owl/owl.ttl b/ontology/owl/owl.ttl index d5bd3f39..dd464601 100644 --- a/ontology/owl/owl.ttl +++ b/ontology/owl/owl.ttl @@ -10,9 +10,9 @@ rdfs:label "uco-owl"@en ; rdfs:comment "This ontology defines SHACL shapes to perform conformance testing of OWL 2 DL. Some of these shapes follow rules specified from the canonical, subtractive parsing process of Section 3 of the OWL 2 mapping to RDF. From the last line of that document's Section 3, 'At the end of this process, the graph G MUST be empty,' anything not strictly matching patterns specified in that section cause the input graph to be non-conformant with OWL 2 DL."@en ; rdfs:seeAlso ; - owl:backwardCompatibleWith uco-owl:1.0.0 ; - owl:priorVersion uco-owl:1.0.0 ; - owl:versionIRI uco-owl:1.1.0 ; + owl:backwardCompatibleWith uco-owl:1.1.0 ; + owl:priorVersion uco-owl:1.1.0 ; + owl:versionIRI uco-owl:1.2.0 ; . uco-owl:Axiom-shape @@ -226,6 +226,7 @@ uco-owl:ObjectProperty-shacl-constraints-shape a sh:SPARQLConstraint ; sh:message "An OWL Object Property must not permit a Literal value via SHACL consraints."@en ; sh:select """ + PREFIX rdf: PREFIX sh: SELECT $this ?value WHERE { diff --git a/ontology/uco/action/action.ttl b/ontology/uco/action/action.ttl index 00ec3ae7..d5a5bf26 100644 --- a/ontology/uco/action/action.ttl +++ b/ontology/uco/action/action.ttl @@ -1,8 +1,8 @@ -# imports: https://ontology.unifiedcyberontology.org/uco/core/1.1.0 -# imports: https://ontology.unifiedcyberontology.org/uco/location/1.1.0 -# imports: https://ontology.unifiedcyberontology.org/uco/pattern/1.1.0 -# imports: https://ontology.unifiedcyberontology.org/uco/types/1.1.0 -# imports: https://ontology.unifiedcyberontology.org/uco/vocabulary/1.1.0 +# imports: https://ontology.unifiedcyberontology.org/uco/core/1.2.0 +# imports: https://ontology.unifiedcyberontology.org/uco/location/1.2.0 +# imports: https://ontology.unifiedcyberontology.org/uco/pattern/1.2.0 +# imports: https://ontology.unifiedcyberontology.org/uco/types/1.2.0 +# imports: https://ontology.unifiedcyberontology.org/uco/vocabulary/1.2.0 @prefix action: . @prefix core: . @@ -20,16 +20,16 @@ a owl:Ontology ; rdfs:label "uco-action"@en ; rdfs:comment "This ontology defines classes and properties for characterizing actions."@en-US ; - owl:backwardCompatibleWith action:1.0.0 ; + owl:backwardCompatibleWith action:1.1.0 ; owl:imports - core:1.1.0 , - location:1.1.0 , - pattern:1.1.0 , - types:1.1.0 , - vocabulary:1.1.0 + core:1.2.0 , + location:1.2.0 , + pattern:1.2.0 , + types:1.2.0 , + vocabulary:1.2.0 ; - owl:priorVersion action:1.0.0 ; - owl:versionIRI action:1.1.0 ; + owl:priorVersion action:1.1.0 ; + owl:versionIRI action:1.2.0 ; . action:Action diff --git a/ontology/uco/action/catalog-v001.xml b/ontology/uco/action/catalog-v001.xml new file mode 100644 index 00000000..3b59aa65 --- /dev/null +++ b/ontology/uco/action/catalog-v001.xml @@ -0,0 +1,12 @@ + + + + + + + + + + + + diff --git a/ontology/uco/analysis/analysis.ttl b/ontology/uco/analysis/analysis.ttl index d98d13f1..11cd5623 100644 --- a/ontology/uco/analysis/analysis.ttl +++ b/ontology/uco/analysis/analysis.ttl @@ -1,4 +1,4 @@ -# imports: https://ontology.unifiedcyberontology.org/uco/action/1.1.0 +# imports: https://ontology.unifiedcyberontology.org/uco/action/1.2.0 @prefix action: . @prefix analysis: . @@ -14,8 +14,10 @@ rdfs:label "uco-analysis"@en ; rdfs:comment "This ontology defines classes and properties for characterizing analytic actions and results."@en-US ; rdfs:seeAlso ; - owl:imports action:1.1.0 ; - owl:versionIRI analysis:1.1.0 ; + owl:backwardCompatibleWith analysis:1.1.0 ; + owl:imports action:1.2.0 ; + owl:priorVersion analysis:1.1.0 ; + owl:versionIRI analysis:1.2.0 ; . analysis:Analysis diff --git a/ontology/uco/analysis/catalog-v001.xml b/ontology/uco/analysis/catalog-v001.xml new file mode 100644 index 00000000..2487f222 --- /dev/null +++ b/ontology/uco/analysis/catalog-v001.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + diff --git a/ontology/uco/configuration/catalog-v001.xml b/ontology/uco/configuration/catalog-v001.xml new file mode 100644 index 00000000..813a8921 --- /dev/null +++ b/ontology/uco/configuration/catalog-v001.xml @@ -0,0 +1,5 @@ + + + + + diff --git a/ontology/uco/configuration/configuration.ttl b/ontology/uco/configuration/configuration.ttl index 14b48d96..6b845e49 100644 --- a/ontology/uco/configuration/configuration.ttl +++ b/ontology/uco/configuration/configuration.ttl @@ -1,4 +1,4 @@ -# imports: https://ontology.unifiedcyberontology.org/uco/core/1.1.0 +# imports: https://ontology.unifiedcyberontology.org/uco/core/1.2.0 @prefix configuration: . @prefix core: . @@ -13,10 +13,10 @@ rdfs:label "uco-configuration"@en ; rdfs:comment "This ontology defines classes and properties for characterizing configurations."@en-US ; rdfs:seeAlso ; - owl:backwardCompatibleWith configuration:1.0.0 ; - owl:imports core:1.1.0 ; - owl:priorVersion configuration:1.0.0 ; - owl:versionIRI configuration:1.1.0 ; + owl:backwardCompatibleWith configuration:1.1.0 ; + owl:imports core:1.2.0 ; + owl:priorVersion configuration:1.1.0 ; + owl:versionIRI configuration:1.2.0 ; . configuration:Configuration diff --git a/ontology/uco/core/catalog-v001.xml b/ontology/uco/core/catalog-v001.xml new file mode 100644 index 00000000..21aa0430 --- /dev/null +++ b/ontology/uco/core/catalog-v001.xml @@ -0,0 +1,4 @@ + + + + diff --git a/ontology/uco/core/core.ttl b/ontology/uco/core/core.ttl index 6ae39b6b..aaa52614 100644 --- a/ontology/uco/core/core.ttl +++ b/ontology/uco/core/core.ttl @@ -10,9 +10,9 @@ a owl:Ontology ; rdfs:label "uco-core"@en ; rdfs:comment "This ontology defines classes and properties that are shared across the various UCO ontologies. At a high-level, the UCO core ontology provides base classes, relationship-oriented classes, content-aggregation classes, and shared classes."@en ; - owl:backwardCompatibleWith core:1.0.0 ; - owl:priorVersion core:1.0.0 ; - owl:versionIRI core:1.1.0 ; + owl:backwardCompatibleWith core:1.1.0 ; + owl:priorVersion core:1.1.0 ; + owl:versionIRI core:1.2.0 ; . core:Annotation @@ -409,15 +409,16 @@ core:UcoThing rdfs:subClassOf owl:Thing ; rdfs:label "UcoThing"@en ; rdfs:comment "UcoThing is the top-level class within UCO."@en ; - rdfs:seeAlso core:UcoThing-identifier-shape ; + rdfs:seeAlso core:UcoThing-identifier-regex-shape ; sh:sparql [ a sh:SPARQLConstraint ; sh:message "UcoThings are required to not be blank nodes."@en ; sh:select """ - PREFIX uco-core: + PREFIX rdfs: + PREFIX core: SELECT $this WHERE { - $this a/rdfs:subClassOf* uco-core:UcoThing . + $this a/rdfs:subClassOf* core:UcoThing . FILTER isBlank ($this) } """ ; @@ -435,10 +436,11 @@ core:UcoThing-identifier-regex-shape rdfs:seeAlso ; sh:message "UcoThings are suggested to end with a UUID."@en ; sh:select ''' - PREFIX uco-core: + PREFIX rdfs: + PREFIX core: SELECT $this WHERE { - $this a/rdfs:subClassOf* uco-core:UcoThing . + $this a/rdfs:subClassOf* core:UcoThing . FILTER ( ! REGEX ( STR($this), diff --git a/ontology/uco/identity/catalog-v001.xml b/ontology/uco/identity/catalog-v001.xml new file mode 100644 index 00000000..666f2eac --- /dev/null +++ b/ontology/uco/identity/catalog-v001.xml @@ -0,0 +1,6 @@ + + + + + + diff --git a/ontology/uco/identity/identity.ttl b/ontology/uco/identity/identity.ttl index 8ba6cf0a..bde0bf28 100644 --- a/ontology/uco/identity/identity.ttl +++ b/ontology/uco/identity/identity.ttl @@ -1,5 +1,5 @@ -# imports: https://ontology.unifiedcyberontology.org/uco/core/1.1.0 -# imports: https://ontology.unifiedcyberontology.org/uco/location/1.1.0 +# imports: https://ontology.unifiedcyberontology.org/uco/core/1.2.0 +# imports: https://ontology.unifiedcyberontology.org/uco/location/1.2.0 @prefix core: . @prefix identity: . @@ -13,13 +13,13 @@ a owl:Ontology ; rdfs:label "uco-identity"@en ; - owl:backwardCompatibleWith identity:1.0.0 ; + owl:backwardCompatibleWith identity:1.1.0 ; owl:imports - core:1.1.0 , - location:1.1.0 + core:1.2.0 , + location:1.2.0 ; - owl:priorVersion identity:1.0.0 ; - owl:versionIRI identity:1.1.0 ; + owl:priorVersion identity:1.1.0 ; + owl:versionIRI identity:1.2.0 ; . identity:AddressFacet diff --git a/ontology/uco/location/catalog-v001.xml b/ontology/uco/location/catalog-v001.xml new file mode 100644 index 00000000..11b8939e --- /dev/null +++ b/ontology/uco/location/catalog-v001.xml @@ -0,0 +1,5 @@ + + + + + diff --git a/ontology/uco/location/location.ttl b/ontology/uco/location/location.ttl index b81688d8..c713430c 100644 --- a/ontology/uco/location/location.ttl +++ b/ontology/uco/location/location.ttl @@ -1,4 +1,4 @@ -# imports: https://ontology.unifiedcyberontology.org/uco/core/1.1.0 +# imports: https://ontology.unifiedcyberontology.org/uco/core/1.2.0 @prefix core: . @prefix location: . @@ -11,10 +11,10 @@ a owl:Ontology ; rdfs:label "uco-location"@en ; - owl:backwardCompatibleWith location:1.0.0 ; - owl:imports core:1.1.0 ; - owl:priorVersion location:1.0.0 ; - owl:versionIRI location:1.1.0 ; + owl:backwardCompatibleWith location:1.1.0 ; + owl:imports core:1.2.0 ; + owl:priorVersion location:1.1.0 ; + owl:versionIRI location:1.2.0 ; . location:GPSCoordinatesFacet diff --git a/ontology/uco/marking/catalog-v001.xml b/ontology/uco/marking/catalog-v001.xml new file mode 100644 index 00000000..a1bc2aa1 --- /dev/null +++ b/ontology/uco/marking/catalog-v001.xml @@ -0,0 +1,5 @@ + + + + + diff --git a/ontology/uco/marking/marking.ttl b/ontology/uco/marking/marking.ttl index e35978de..ea1d6327 100644 --- a/ontology/uco/marking/marking.ttl +++ b/ontology/uco/marking/marking.ttl @@ -1,4 +1,4 @@ -# imports: https://ontology.unifiedcyberontology.org/uco/core/1.1.0 +# imports: https://ontology.unifiedcyberontology.org/uco/core/1.2.0 @prefix core: . @prefix marking: . @@ -12,10 +12,10 @@ a owl:Ontology ; rdfs:label "uco-marking"@en ; rdfs:comment "Data markings represent restrictions, permissions, and other guidance for how data can be used and shared."@en ; - owl:backwardCompatibleWith marking:1.0.0 ; - owl:imports core:1.1.0 ; - owl:priorVersion marking:1.0.0 ; - owl:versionIRI marking:1.1.0 ; + owl:backwardCompatibleWith marking:1.1.0 ; + owl:imports core:1.2.0 ; + owl:priorVersion marking:1.1.0 ; + owl:versionIRI marking:1.2.0 ; . marking:GranularMarking diff --git a/ontology/uco/master/catalog-v001.xml b/ontology/uco/master/catalog-v001.xml new file mode 100644 index 00000000..5145a6db --- /dev/null +++ b/ontology/uco/master/catalog-v001.xml @@ -0,0 +1,23 @@ + + + + + + + + + + + + + + + + + + + + + + + diff --git a/ontology/uco/master/uco.ttl b/ontology/uco/master/uco.ttl index eb74b196..2b2ca471 100644 --- a/ontology/uco/master/uco.ttl +++ b/ontology/uco/master/uco.ttl @@ -1,20 +1,20 @@ -# imports: https://ontology.unifiedcyberontology.org/co/1.1.0 -# imports: https://ontology.unifiedcyberontology.org/owl/1.1.0 -# imports: https://ontology.unifiedcyberontology.org/uco/action/1.1.0 -# imports: https://ontology.unifiedcyberontology.org/uco/analysis/1.1.0 -# imports: https://ontology.unifiedcyberontology.org/uco/configuration/1.1.0 -# imports: https://ontology.unifiedcyberontology.org/uco/core/1.1.0 -# imports: https://ontology.unifiedcyberontology.org/uco/identity/1.1.0 -# imports: https://ontology.unifiedcyberontology.org/uco/location/1.1.0 -# imports: https://ontology.unifiedcyberontology.org/uco/marking/1.1.0 -# imports: https://ontology.unifiedcyberontology.org/uco/observable/1.1.0 -# imports: https://ontology.unifiedcyberontology.org/uco/pattern/1.1.0 -# imports: https://ontology.unifiedcyberontology.org/uco/role/1.1.0 -# imports: https://ontology.unifiedcyberontology.org/uco/time/1.1.0 -# imports: https://ontology.unifiedcyberontology.org/uco/tool/1.1.0 -# imports: https://ontology.unifiedcyberontology.org/uco/types/1.1.0 -# imports: https://ontology.unifiedcyberontology.org/uco/victim/1.1.0 -# imports: https://ontology.unifiedcyberontology.org/uco/vocabulary/1.1.0 +# imports: https://ontology.unifiedcyberontology.org/co/1.2.0 +# imports: https://ontology.unifiedcyberontology.org/owl/1.2.0 +# imports: https://ontology.unifiedcyberontology.org/uco/action/1.2.0 +# imports: https://ontology.unifiedcyberontology.org/uco/analysis/1.2.0 +# imports: https://ontology.unifiedcyberontology.org/uco/configuration/1.2.0 +# imports: https://ontology.unifiedcyberontology.org/uco/core/1.2.0 +# imports: https://ontology.unifiedcyberontology.org/uco/identity/1.2.0 +# imports: https://ontology.unifiedcyberontology.org/uco/location/1.2.0 +# imports: https://ontology.unifiedcyberontology.org/uco/marking/1.2.0 +# imports: https://ontology.unifiedcyberontology.org/uco/observable/1.2.0 +# imports: https://ontology.unifiedcyberontology.org/uco/pattern/1.2.0 +# imports: https://ontology.unifiedcyberontology.org/uco/role/1.2.0 +# imports: https://ontology.unifiedcyberontology.org/uco/time/1.2.0 +# imports: https://ontology.unifiedcyberontology.org/uco/tool/1.2.0 +# imports: https://ontology.unifiedcyberontology.org/uco/types/1.2.0 +# imports: https://ontology.unifiedcyberontology.org/uco/victim/1.2.0 +# imports: https://ontology.unifiedcyberontology.org/uco/vocabulary/1.2.0 @prefix owl: . @prefix rdf: . @@ -25,28 +25,28 @@ a owl:Ontology ; rdfs:label "uco-master"@en ; - owl:backwardCompatibleWith ; + owl:backwardCompatibleWith ; owl:imports - , - , - , - , - , - , - , - , - , - , - , - , - , - , - , - , - + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + ; - owl:priorVersion ; - owl:versionIRI ; - owl:versionInfo "1.1.0" ; + owl:priorVersion ; + owl:versionIRI ; + owl:versionInfo "1.2.0" ; . diff --git a/ontology/uco/observable/catalog-v001.xml b/ontology/uco/observable/catalog-v001.xml new file mode 100644 index 00000000..bc82826c --- /dev/null +++ b/ontology/uco/observable/catalog-v001.xml @@ -0,0 +1,15 @@ + + + + + + + + + + + + + + + diff --git a/ontology/uco/observable/observable.ttl b/ontology/uco/observable/observable.ttl index 5266d99c..eac9609a 100644 --- a/ontology/uco/observable/observable.ttl +++ b/ontology/uco/observable/observable.ttl @@ -1,10 +1,10 @@ -# imports: https://ontology.unifiedcyberontology.org/uco/action/1.1.0 -# imports: https://ontology.unifiedcyberontology.org/uco/configuration/1.1.0 -# imports: https://ontology.unifiedcyberontology.org/uco/core/1.1.0 -# imports: https://ontology.unifiedcyberontology.org/uco/identity/1.1.0 -# imports: https://ontology.unifiedcyberontology.org/uco/location/1.1.0 -# imports: https://ontology.unifiedcyberontology.org/uco/types/1.1.0 -# imports: https://ontology.unifiedcyberontology.org/uco/vocabulary/1.1.0 +# imports: https://ontology.unifiedcyberontology.org/uco/action/1.2.0 +# imports: https://ontology.unifiedcyberontology.org/uco/configuration/1.2.0 +# imports: https://ontology.unifiedcyberontology.org/uco/core/1.2.0 +# imports: https://ontology.unifiedcyberontology.org/uco/identity/1.2.0 +# imports: https://ontology.unifiedcyberontology.org/uco/location/1.2.0 +# imports: https://ontology.unifiedcyberontology.org/uco/types/1.2.0 +# imports: https://ontology.unifiedcyberontology.org/uco/vocabulary/1.2.0 @prefix action: . @prefix co: . @@ -24,18 +24,18 @@ a owl:Ontology ; rdfs:label "uco-observable"@en ; - owl:backwardCompatibleWith observable:1.0.0 ; + owl:backwardCompatibleWith observable:1.1.0 ; owl:imports - action:1.1.0 , - configuration:1.1.0 , - core:1.1.0 , - identity:1.1.0 , - location:1.1.0 , - types:1.1.0 , - vocabulary:1.1.0 + action:1.2.0 , + configuration:1.2.0 , + core:1.2.0 , + identity:1.2.0 , + location:1.2.0 , + types:1.2.0 , + vocabulary:1.2.0 ; - owl:priorVersion observable:1.0.0 ; - owl:versionIRI observable:1.1.0 ; + owl:priorVersion observable:1.1.0 ; + owl:versionIRI observable:1.2.0 ; . observable:API @@ -4480,25 +4480,24 @@ observable:MobileDeviceFacet sh:datatype xsd:string ; sh:maxCount "1"^^xsd:integer ; sh:nodeKind sh:Literal ; - sh:path observable:IMEI ; + sh:path observable:bluetoothDeviceName ; ] , [ sh:datatype xsd:string ; sh:maxCount "1"^^xsd:integer ; sh:nodeKind sh:Literal ; - sh:path observable:bluetoothDeviceName ; + sh:path observable:keypadUnlockCode ; ] , [ sh:datatype xsd:string ; sh:maxCount "1"^^xsd:integer ; sh:nodeKind sh:Literal ; - sh:path observable:keypadUnlockCode ; + sh:path observable:network ; ] , [ sh:datatype xsd:string ; - sh:maxCount "1"^^xsd:integer ; sh:nodeKind sh:Literal ; - sh:path observable:network ; + sh:path observable:IMEI ; ] ; sh:targetClass observable:MobileDeviceFacet ; @@ -6215,6 +6214,23 @@ observable:StorageMedium sh:targetClass observable:StorageMedium ; . +observable:StorageMediumFacet + a + owl:Class , + sh:NodeShape + ; + rdfs:subClassOf core:Facet ; + rdfs:label "StorageMediumFacet"@en ; + rdfs:comment "A storage medium facet is a grouping of characteristics unique to a the storage capabilities of a piece of equipment or a mechanism designed to serve a special purpose or perform a special function."@en ; + sh:property [ + sh:datatype xsd:integer ; + sh:maxCount "1"^^xsd:integer ; + sh:nodeKind sh:Literal ; + sh:path observable:totalStorageCapacityInBytes ; + ] ; + sh:targetClass observable:StorageMediumFacet ; + . + observable:SymbolicLink a owl:Class , @@ -8865,6 +8881,12 @@ observable:WindowsThreadFacet sh:nodeKind sh:Literal ; sh:path observable:creationTime ; ] , + [ + sh:datatype xsd:dateTime ; + sh:maxCount "1"^^xsd:integer ; + sh:nodeKind sh:Literal ; + sh:path observable:observableCreatedTime ; + ] , [ sh:datatype xsd:hexBinary ; sh:nodeKind sh:Literal ; @@ -10191,11 +10213,26 @@ observable:creationFlags . observable:creationTime - a owl:DatatypeProperty ; + a + owl:DatatypeProperty , + owl:DeprecatedProperty + ; rdfs:label "creationTime"@en ; rdfs:range xsd:dateTime ; . +observable:creationTime-deprecation-shape + a sh:NodeShape ; + sh:property [ + a sh:PropertyShape ; + sh:maxCount "0"^^xsd:integer ; + sh:message "observable:creationTime is deprecated, and will be an error to use in UCO 2.0.0. observable:observableCreatedTime should be used instead."@en ; + sh:path observable:creationTime ; + sh:severity sh:Warning ; + ] ; + sh:targetSubjectsOf observable:creationTime ; + . + observable:creator a owl:ObjectProperty ; rdfs:label "creator"@en ; @@ -13292,6 +13329,13 @@ observable:totalSpace rdfs:range xsd:integer ; . +observable:totalStorageCapacityInBytes + a owl:DatatypeProperty ; + rdfs:label "totalStorageCapacityInBytes"@en ; + rdfs:comment "The maximum number of bytes that can be stored on a storage device."@en ; + rdfs:range xsd:integer ; + . + observable:triggerBeginTime a owl:DatatypeProperty ; rdfs:label "triggerBeginTime"@en ; diff --git a/ontology/uco/pattern/catalog-v001.xml b/ontology/uco/pattern/catalog-v001.xml new file mode 100644 index 00000000..ec640714 --- /dev/null +++ b/ontology/uco/pattern/catalog-v001.xml @@ -0,0 +1,5 @@ + + + + + diff --git a/ontology/uco/pattern/pattern.ttl b/ontology/uco/pattern/pattern.ttl index ae30008e..de7163c8 100644 --- a/ontology/uco/pattern/pattern.ttl +++ b/ontology/uco/pattern/pattern.ttl @@ -1,4 +1,4 @@ -# imports: https://ontology.unifiedcyberontology.org/uco/core/1.1.0 +# imports: https://ontology.unifiedcyberontology.org/uco/core/1.2.0 @prefix core: . @prefix owl: . @@ -11,10 +11,10 @@ a owl:Ontology ; rdfs:label "uco-pattern"@en ; - owl:backwardCompatibleWith pattern:1.0.0 ; - owl:imports core:1.1.0 ; - owl:priorVersion pattern:1.0.0 ; - owl:versionIRI pattern:1.1.0 ; + owl:backwardCompatibleWith pattern:1.1.0 ; + owl:imports core:1.2.0 ; + owl:priorVersion pattern:1.1.0 ; + owl:versionIRI pattern:1.2.0 ; . pattern:LogicalPattern diff --git a/ontology/uco/role/catalog-v001.xml b/ontology/uco/role/catalog-v001.xml new file mode 100644 index 00000000..2580b041 --- /dev/null +++ b/ontology/uco/role/catalog-v001.xml @@ -0,0 +1,5 @@ + + + + + diff --git a/ontology/uco/role/role.ttl b/ontology/uco/role/role.ttl index f7e12b19..4537d315 100644 --- a/ontology/uco/role/role.ttl +++ b/ontology/uco/role/role.ttl @@ -1,4 +1,4 @@ -# imports: https://ontology.unifiedcyberontology.org/uco/core/1.1.0 +# imports: https://ontology.unifiedcyberontology.org/uco/core/1.2.0 @prefix core: . @prefix owl: . @@ -11,10 +11,10 @@ a owl:Ontology ; rdfs:label "uco-role"@en ; - owl:backwardCompatibleWith role:1.0.0 ; - owl:imports core:1.1.0 ; - owl:priorVersion role:1.0.0 ; - owl:versionIRI role:1.1.0 ; + owl:backwardCompatibleWith role:1.1.0 ; + owl:imports core:1.2.0 ; + owl:priorVersion role:1.1.0 ; + owl:versionIRI role:1.2.0 ; . role:BenevolentRole diff --git a/ontology/uco/time/catalog-v001.xml b/ontology/uco/time/catalog-v001.xml new file mode 100644 index 00000000..08318a2f --- /dev/null +++ b/ontology/uco/time/catalog-v001.xml @@ -0,0 +1,5 @@ + + + + + diff --git a/ontology/uco/time/time.ttl b/ontology/uco/time/time.ttl index 5d35a5a2..ef6adc72 100644 --- a/ontology/uco/time/time.ttl +++ b/ontology/uco/time/time.ttl @@ -1,4 +1,4 @@ -# imports: https://ontology.unifiedcyberontology.org/uco/core/1.1.0 +# imports: https://ontology.unifiedcyberontology.org/uco/core/1.2.0 @prefix core: . @prefix owl: . @@ -10,10 +10,10 @@ a owl:Ontology ; rdfs:label "uco-time"@en ; - owl:backwardCompatibleWith time:1.0.0 ; - owl:imports core:1.1.0 ; - owl:priorVersion time:1.0.0 ; - owl:versionIRI time:1.1.0 ; + owl:backwardCompatibleWith time:1.1.0 ; + owl:imports core:1.2.0 ; + owl:priorVersion time:1.1.0 ; + owl:versionIRI time:1.2.0 ; . time:Time diff --git a/ontology/uco/tool/catalog-v001.xml b/ontology/uco/tool/catalog-v001.xml new file mode 100644 index 00000000..906c7cad --- /dev/null +++ b/ontology/uco/tool/catalog-v001.xml @@ -0,0 +1,8 @@ + + + + + + + + diff --git a/ontology/uco/tool/tool.ttl b/ontology/uco/tool/tool.ttl index 09ea91cf..4439e439 100644 --- a/ontology/uco/tool/tool.ttl +++ b/ontology/uco/tool/tool.ttl @@ -1,5 +1,5 @@ -# imports: https://ontology.unifiedcyberontology.org/uco/configuration/1.1.0 -# imports: https://ontology.unifiedcyberontology.org/uco/identity/1.1.0 +# imports: https://ontology.unifiedcyberontology.org/uco/configuration/1.2.0 +# imports: https://ontology.unifiedcyberontology.org/uco/identity/1.2.0 @prefix configuration: . @prefix core: . @@ -14,13 +14,13 @@ a owl:Ontology ; rdfs:label "uco-tool"@en ; - owl:backwardCompatibleWith tool:1.0.0 ; + owl:backwardCompatibleWith tool:1.1.0 ; owl:imports - configuration:1.1.0 , - identity:1.1.0 + configuration:1.2.0 , + identity:1.2.0 ; - owl:priorVersion tool:1.0.0 ; - owl:versionIRI tool:1.1.0 ; + owl:priorVersion tool:1.1.0 ; + owl:versionIRI tool:1.2.0 ; . tool:AnalyticTool diff --git a/ontology/uco/types/catalog-v001.xml b/ontology/uco/types/catalog-v001.xml new file mode 100644 index 00000000..95833c1a --- /dev/null +++ b/ontology/uco/types/catalog-v001.xml @@ -0,0 +1,9 @@ + + + + + + + + + diff --git a/ontology/uco/types/types.ttl b/ontology/uco/types/types.ttl index 84c03182..8dc0e9a1 100644 --- a/ontology/uco/types/types.ttl +++ b/ontology/uco/types/types.ttl @@ -1,6 +1,6 @@ -# imports: https://ontology.unifiedcyberontology.org/co/1.1.0 -# imports: https://ontology.unifiedcyberontology.org/uco/core/1.1.0 -# imports: https://ontology.unifiedcyberontology.org/uco/vocabulary/1.1.0 +# imports: https://ontology.unifiedcyberontology.org/co/1.2.0 +# imports: https://ontology.unifiedcyberontology.org/uco/core/1.2.0 +# imports: https://ontology.unifiedcyberontology.org/uco/vocabulary/1.2.0 @prefix co: . @prefix core: . @@ -16,14 +16,14 @@ a owl:Ontology ; rdfs:label "uco-types"@en ; - owl:backwardCompatibleWith types:1.0.0 ; + owl:backwardCompatibleWith types:1.1.0 ; owl:imports - uco-co:1.1.0 , - core:1.1.0 , - vocabulary:1.1.0 + uco-co:1.2.0 , + core:1.2.0 , + vocabulary:1.2.0 ; - owl:priorVersion types:1.0.0 ; - owl:versionIRI types:1.1.0 ; + owl:priorVersion types:1.1.0 ; + owl:versionIRI types:1.2.0 ; . types:ControlledDictionary @@ -143,6 +143,10 @@ types:Hash "SHA1"^^vocabulary:HashNameVocab "SHA224"^^vocabulary:HashNameVocab "SHA256"^^vocabulary:HashNameVocab + "SHA3-224"^^vocabulary:HashNameVocab + "SHA3-256"^^vocabulary:HashNameVocab + "SHA3-384"^^vocabulary:HashNameVocab + "SHA3-512"^^vocabulary:HashNameVocab "SHA384"^^vocabulary:HashNameVocab "SHA512"^^vocabulary:HashNameVocab "SSDEEP"^^vocabulary:HashNameVocab diff --git a/ontology/uco/victim/catalog-v001.xml b/ontology/uco/victim/catalog-v001.xml new file mode 100644 index 00000000..cdc6d411 --- /dev/null +++ b/ontology/uco/victim/catalog-v001.xml @@ -0,0 +1,6 @@ + + + + + + diff --git a/ontology/uco/victim/victim.ttl b/ontology/uco/victim/victim.ttl index 2d1c0a81..1e6586f9 100644 --- a/ontology/uco/victim/victim.ttl +++ b/ontology/uco/victim/victim.ttl @@ -1,4 +1,4 @@ -# imports: https://ontology.unifiedcyberontology.org/uco/role/1.1.0 +# imports: https://ontology.unifiedcyberontology.org/uco/role/1.2.0 @prefix owl: . @prefix rdf: . @@ -11,10 +11,10 @@ a owl:Ontology ; rdfs:label "uco-victim"@en ; - owl:backwardCompatibleWith victim:1.0.0 ; - owl:imports role:1.1.0 ; - owl:priorVersion victim:1.0.0 ; - owl:versionIRI victim:1.1.0 ; + owl:backwardCompatibleWith victim:1.1.0 ; + owl:imports role:1.2.0 ; + owl:priorVersion victim:1.1.0 ; + owl:versionIRI victim:1.2.0 ; . victim:Victim diff --git a/ontology/uco/vocabulary/catalog-v001.xml b/ontology/uco/vocabulary/catalog-v001.xml new file mode 100644 index 00000000..bca77538 --- /dev/null +++ b/ontology/uco/vocabulary/catalog-v001.xml @@ -0,0 +1,4 @@ + + + + diff --git a/ontology/uco/vocabulary/vocabulary.ttl b/ontology/uco/vocabulary/vocabulary.ttl index fc3c1325..a376011b 100644 --- a/ontology/uco/vocabulary/vocabulary.ttl +++ b/ontology/uco/vocabulary/vocabulary.ttl @@ -7,9 +7,9 @@ a owl:Ontology ; rdfs:label "uco-vocabularies"@en ; - owl:backwardCompatibleWith vocabulary:1.0.0 ; - owl:priorVersion vocabulary:1.0.0 ; - owl:versionIRI vocabulary:1.1.0 ; + owl:backwardCompatibleWith vocabulary:1.1.0 ; + owl:priorVersion vocabulary:1.1.0 ; + owl:versionIRI vocabulary:1.2.0 ; . vocabulary:AccountTypeVocab @@ -614,6 +614,10 @@ vocabulary:HashNameVocab "SHA1"^^vocabulary:HashNameVocab "SHA224"^^vocabulary:HashNameVocab "SHA256"^^vocabulary:HashNameVocab + "SHA3-224"^^vocabulary:HashNameVocab + "SHA3-256"^^vocabulary:HashNameVocab + "SHA3-384"^^vocabulary:HashNameVocab + "SHA3-512"^^vocabulary:HashNameVocab "SHA384"^^vocabulary:HashNameVocab "SHA512"^^vocabulary:HashNameVocab "SSDEEP"^^vocabulary:HashNameVocab @@ -781,6 +785,7 @@ vocabulary:ObservableObjectRelationshipVocab "Sent_Via_Upload"^^vocabulary:ObservableObjectRelationshipVocab "Set_From"^^vocabulary:ObservableObjectRelationshipVocab "Set_To"^^vocabulary:ObservableObjectRelationshipVocab + "Signed_By"^^vocabulary:ObservableObjectRelationshipVocab "Sub-domain_Of"^^vocabulary:ObservableObjectRelationshipVocab "Supra-domain_Of"^^vocabulary:ObservableObjectRelationshipVocab "Suspended"^^vocabulary:ObservableObjectRelationshipVocab diff --git a/tests/requirements.txt b/requirements.txt similarity index 100% rename from tests/requirements.txt rename to requirements.txt diff --git a/src/create-catalog-v001.xml.py b/src/create-catalog-v001.xml.py new file mode 100644 index 00000000..a3b0be5b --- /dev/null +++ b/src/create-catalog-v001.xml.py @@ -0,0 +1,366 @@ +#!/usr/bin/env python3 + +# Portions of this file contributed by NIST are governed by the following +# statement: +# +# This software was developed at the National Institute of Standards +# and Technology by employees of the Federal Government in the course +# of their official duties. Pursuant to title 17 Section 105 of the +# United States Code this software is not subject to copyright +# protection and is in the public domain. NIST assumes no +# responsibility whatsoever for its use by other parties, and makes +# no guarantees, expressed or implied, about its quality, +# reliability, or any other characteristic. +# +# We would appreciate acknowledgement if the software is used. + +""" +This program constructs a catalog-v001.xml file, initially implemented +to satisfy local-file needs of the Protégé ontology editor. The +resulting catalog file lets a user of the XML file interact with their +ontology, even spread across multiple files, without requesting network +resources. This is beneficial, for instance, in tracking local edits to +resources otherwise stored online. It is also beneficial when networked +resources are not available, such as due to service interruptions or +link rot. + +One catalog-v001.xml file will generally support only its housing +directory in a source code hierarchy. The catalog file is built to make +relative path references to every OWL-imported IRI in the transitive +import closure of all of the ontology graph files in the directory. +Once the catalog file is generated, a user should be able to open a +sibling graph file in the same directory and have Protégé load without +making network requests for ontology graphs. +""" + +__version__ = "0.1.0" + +import argparse +import csv +import logging +import os +import xml.etree.ElementTree as ETree +from pathlib import Path +from typing import Dict, List, Set, Tuple +from xml.dom import minidom + +from rdflib import OWL, RDF, Graph, URIRef + +NS_OWL = OWL +NS_RDF = RDF + + +# XML prolog, as generated by Protégé. +XML_VERSION_INFO = '' + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("--debug", action="store_true") + parser.add_argument( + "--catalog-xml", + help="A generated catalog-v001.xml file for some dependent or imported ontology. This should be supplied for ontologies not covered by the dependency_files_tsv argument, nor meant to be crawled upon using domain_directories_tsv. (For instance, this could be used to import relative file references for ontologies tracked as Git submodules.)", + action="append", + ) + # "x" mode - exclusive creation. + # https://docs.python.org/3/library/functions.html#open + parser.add_argument("out_xml", type=argparse.FileType("x")) + parser.add_argument( + "domain_directories_tsv", + help="A two-column file, with column 1 being a string prefix in-common to ontology prefix IRIs, and column 2 being a file system directory relative to top_srcdir that is the root directory housing that ontology's files. Directories specified in this file will be recursively walked to discover ontology graph files. This file may be empty, but it must exist.", + type=argparse.FileType("r"), + ) + parser.add_argument( + "dependency_files_tsv", + help="A two-column file, with column 1 being an ontology reference IRI (ontology IRI or version IRI), and column 2 being a local, version-controlled file relative to top_srcdir that houses the corresponding ontology data. This file may be empty, but it must exist.", + type=argparse.FileType("r"), + ) + parser.add_argument("top_srcdir", help="The root directory of the Git repository for this ontology. In the two TSV arguments, the variable top_srcdir is substituted with this value.") + parser.add_argument( + "in_ttl", + help="Input graph files. Due to the target use case of Protégé Catalog files, these are required to be in the same directory.", + nargs="+", + ) + args = parser.parse_args() + + logging.basicConfig(level=logging.DEBUG if args.debug else logging.INFO) + + top_srcdir_abspath = Path(args.top_srcdir).resolve() + if not top_srcdir_abspath.exists(): + raise FileNotFoundError(args.top_srcdir) + if not top_srcdir_abspath.is_dir(): + raise NotADirectoryError(args.top_srcdir) + logging.debug("top_srcdir_abspath = %r.", top_srcdir_abspath) + + focus_graph = Graph() + + focus_graph_srcdir_abspaths: Set[Path] = set() + for in_ttl in args.in_ttl: + focus_graph_abspath = Path(in_ttl).resolve() + focus_graph.parse(str(focus_graph_abspath)) + focus_graph_srcdir_abspaths.add(focus_graph_abspath.parent) + if len(focus_graph_srcdir_abspaths) > 1: + for focus_graph_srcdir_abspath_no, focus_graph_srcdir_abspath in enumerate( + sorted(focus_graph_srcdir_abspaths) + ): + logging.error( + "%d: %s", + 1 + focus_graph_srcdir_abspath_no, + str(focus_graph_srcdir_abspath), + ) + raise ValueError( + "Input graphs are required to be in the same directory. Found them in %d directories." + % len(focus_graph_srcdir_abspaths) + ) + focus_graph_srcdir_abspath = sorted(focus_graph_srcdir_abspaths)[0] + + top_srcdir_relpath = Path( + os.path.relpath(top_srcdir_abspath, focus_graph_srcdir_abspath) + ) + logging.debug("top_srcdir_relpath = %r.", top_srcdir_relpath) + + # Determine focus ontology IRIs. + n_focus_ontologies: Set[URIRef] = set() + for triple in focus_graph.triples((None, NS_RDF.type, NS_OWL.Ontology)): + if isinstance(triple[0], URIRef): + n_focus_ontologies.add(triple[0]) + if len(n_focus_ontologies) < 1: + raise ValueError("Found no focus ontology IRI.") + + # Free focus-graph memory. + del focus_graph + + # Read TSV to get domain prefixes' housing directories. + ontology_string_prefix_to_domain_directory: Dict[str, Path] = dict() + reader = csv.reader(args.domain_directories_tsv, delimiter="\t") + for row in reader: + ontology_string_prefix = row[0] + domain_directory_str = row[1].replace("${top_srcdir}", str(top_srcdir_abspath)) + domain_directory = Path(domain_directory_str) + if not domain_directory.exists(): + raise FileNotFoundError(domain_directory_str) + if not domain_directory.is_dir(): + raise NotADirectoryError(domain_directory_str) + ontology_string_prefix_to_domain_directory[ + ontology_string_prefix + ] = domain_directory + logging.debug( + "ontology_string_prefix_to_domain_directory = %r.", + ontology_string_prefix_to_domain_directory, + ) + + # Walk domain directories to associate ontology reference IRIs with backing files, and to build imports graph. + # + # Definition, possibly specialized to just this script: + # An "ontology reference IRI" is either an ontology IRI or a versionIRI of an ontology. + imports_graph = Graph() + n_ontology_reference_to_backing_file: Dict[URIRef, Path] = dict() + + def _load_graph(graph_file_path: Path) -> None: + tmp_graph = Graph() + logging.debug("graph_file_path = %r.", graph_file_path) + tmp_graph.parse(str(graph_file_path)) + for triple in tmp_graph.triples((None, NS_RDF.type, NS_OWL.Ontology)): + assert isinstance(triple[0], URIRef) + n_ontology_reference_to_backing_file[triple[0]] = graph_file_path + imports_graph.add(triple) + for triple in tmp_graph.triples((None, NS_OWL.imports, None)): + imports_graph.add(triple) + for triple in tmp_graph.triples((None, NS_OWL.versionIRI, None)): + assert isinstance(triple[2], URIRef) + n_ontology_reference_to_backing_file[triple[2]] = graph_file_path + imports_graph.add(triple) + + # Do deep walk for domain directories. + for domain_directory in ontology_string_prefix_to_domain_directory.values(): + for dirpath, dirnames, filenames in os.walk(str(domain_directory)): + for filename in filenames: + # Skip build files (syntax normalization checks). + if filename.startswith("."): + continue + # Restrict to Turtle files. + if not filename.endswith(".ttl"): + continue + dirpath_path = Path(dirpath) + graph_filepath = dirpath_path / filename + _load_graph(graph_filepath) + # Do direct imports from dependency file map. + reader = csv.reader(args.dependency_files_tsv, delimiter="\t") + for row in reader: + logging.debug("row = %r.", row) + n_ontology_reference = URIRef(row[0]) + graph_file_name = row[1].replace("${top_srcdir}", str(top_srcdir_abspath)) + graph_file_path = Path(graph_file_name) + if not graph_file_path.exists(): + raise FileNotFoundError(graph_file_path) + if graph_file_path.is_dir(): + raise IsADirectoryError(graph_file_path) + n_ontology_reference_to_backing_file[n_ontology_reference] = graph_file_path + _load_graph(graph_file_path) + # Inherit prior catalog files. + catalog_paths: Set[Path] = set() + if args.catalog_xml: + for catalog_file_name in args.catalog_xml: + catalog_path = Path(catalog_file_name).resolve() + logging.debug("catalog_path = %r.", catalog_path) + if not catalog_path.exists(): + raise FileNotFoundError(catalog_file_name) + if catalog_path.name != "catalog-v001.xml": + logging.error("catalog_file_name = %r.", catalog_file_name) + raise FileNotFoundError( + 'Expecting catalog file to be named "catalog-v001.xml".' + ) + catalog_paths.add(catalog_path) + for catalog_path in sorted(catalog_paths): + logging.debug("catalog_path = %r.", catalog_path) + catalogued_directory_path = catalog_path.parent + # Load graph files accompanying this catalog-v001.xml. + for file_path in catalogued_directory_path.iterdir(): + file_basename = file_path.name + # Skip build files (syntax normalization checks). + if file_basename.startswith("."): + continue + # Restrict to Turtle files. + if not file_basename.endswith(".ttl"): + continue + _load_graph(catalog_path) + # Use catalog-v001.xml to find further graph files. + tree = ETree.parse(catalog_path) + for child in tree.getroot(): + logging.debug("child.attrib = %r.", child.attrib) + logging.debug("child.tag = %r.", child.tag) + if child.tag != "{urn:oasis:names:tc:entity:xmlns:xml:catalog}uri": + continue + if child.attrib["uri"].startswith("http:"): + continue + if child.attrib["uri"].startswith("https:"): + continue + if child.attrib["uri"].startswith("urn:"): + continue + n_ontology_reference = URIRef(child.attrib["name"]) + backing_ontology_path = catalogued_directory_path / child.attrib["uri"] + logging.debug("backing_ontology_path = %r.", backing_ontology_path) + if not backing_ontology_path.exists(): + logging.info( + "catalogued_directory_path = %r.", catalogued_directory_path + ) + logging.info('child.attrib["uri"] = %r.', child.attrib["uri"]) + raise FileNotFoundError("Unable to find referenced ontology file.") + if not backing_ontology_path.is_file(): + logging.info( + "catalogued_directory_path = %r.", catalogued_directory_path + ) + logging.info('child.attrib["uri"] = %r.', child.attrib["uri"]) + raise ValueError("Referenced ontology file path is not regular file.") + n_ontology_reference_to_backing_file[ + n_ontology_reference + ] = backing_ontology_path.resolve() + _load_graph(backing_ontology_path) + + logging.debug("len(imports_graph) = %d.", len(imports_graph)) + if args.debug: + logging.debug("n_ontology_reference_to_backing_file:") + for n_ontology_reference in sorted(n_ontology_reference_to_backing_file.keys()): + logging.debug( + "%r -> %r.", + n_ontology_reference, + n_ontology_reference_to_backing_file[n_ontology_reference], + ) + + n_imported_iri_to_relative_backing_path: Dict[URIRef, Path] = dict() + + def _map_n_ontology_reference(n_ontology_reference: URIRef) -> None: + logging.debug("n_ontology_reference = %r.", n_ontology_reference) + ontology_reference_backing_file_abspath = n_ontology_reference_to_backing_file[ + n_ontology_reference + ] + ontology_reference_backing_file_relpath = Path( + os.path.relpath( + ontology_reference_backing_file_abspath, focus_graph_srcdir_abspath + ) + ) + n_imported_iri_to_relative_backing_path[ + n_ontology_reference + ] = ontology_reference_backing_file_relpath + n_imported_iris: Set[URIRef] = set() + for result in imports_graph.query( + """\ +SELECT ?nImportIRI +WHERE { + { + ?nOntologyReference + owl:imports+ ?nImportIRI ; + . + } + UNION + { + ?nOntology + owl:imports+ ?nImportIRI ; + owl:versionIRI ?nOntologyReference ; + . + } +} +""", + initBindings={"nOntologyReference": n_ontology_reference}, + ): + assert isinstance(result[0], URIRef) + n_imported_iri = result[0] + n_imported_iris.add(n_imported_iri) + # Handle base case - cut mapped nodes. + logging.debug("n_imported_iris = %r.", n_imported_iris) + n_imported_unvisited_iris = n_imported_iris - { + x for x in n_imported_iri_to_relative_backing_path.keys() + } + logging.debug("n_imported_unvisited_iris = %r.", n_imported_unvisited_iris) + # Recurse, because owl:imports could have a versionIRI as its object. + for n_imported_unvisited_iri in n_imported_unvisited_iris: + _map_n_ontology_reference(n_imported_unvisited_iri) + + for n_focus_ontology in n_focus_ontologies: + _map_n_ontology_reference(n_focus_ontology) + + if args.debug: + logging.debug("n_imported_iri_to_relative_backing_path:") + for n_imported_iri in sorted(n_imported_iri_to_relative_backing_path.keys()): + logging.debug( + "* %r -> %r.", + n_imported_iri, + n_imported_iri_to_relative_backing_path[n_imported_iri], + ) + + # Create catalog XML tree. + xml_root = ETree.Element("catalog") + + # Mimic attributes for the root node from exemplar generated by Protégé. + xml_root.attrib = { + "prefer": "public", + "xmlns": "urn:oasis:names:tc:entity:xmlns:xml:catalog", + } + # Sort catalog entries by relative file path, again mimicing Protégé behavior. + catalog_entries: List[Tuple[str, str]] = sorted( + [ + ( + str(n_imported_iri_to_relative_backing_path[n_ontology_reference]), + str(n_ontology_reference), + ) + for n_ontology_reference in n_imported_iri_to_relative_backing_path.keys() + ] + ) + for catalog_entry in catalog_entries: + e_child = ETree.SubElement(xml_root, "uri") + e_child.attrib = { + "id": "User Entered Import Resolution", + "uri": catalog_entry[0], + "name": catalog_entry[1], + } + xml_tree_string = minidom.parseString( + ETree.tostring(xml_root, encoding="utf-8", method="xml").decode("utf-8") + ).toprettyxml(indent=" ") + args.out_xml.write(f"{XML_VERSION_INFO}\n") + args.out_xml.write(f"{xml_tree_string[23:]}") + + return + + +if __name__ == "__main__": + main() diff --git a/src/review.mk b/src/review.mk index f4176d55..999c7c26 100644 --- a/src/review.mk +++ b/src/review.mk @@ -26,7 +26,8 @@ check_reference_basenames := $(foreach ttl_basename,$(ttl_basenames),.check-$(tt check_targets := $(foreach ttl_basename,$(ttl_basenames),check-$(ttl_basename)) all: \ - $(check_reference_basenames) + $(check_reference_basenames) \ + catalog-v001.xml .check-%.ttl: \ %.ttl \ @@ -39,8 +40,24 @@ all: \ --target-format turtle mv $@_ $@ +catalog-v001.xml: \ + $(top_srcdir)/.venv.done.log \ + $(top_srcdir)/etc/domain_directories.tsv \ + $(top_srcdir)/etc/dependency_files.tsv \ + $(top_srcdir)/src/create-catalog-v001.xml.py + rm -f _$@ + source $(top_srcdir)/venv/bin/activate \ + && python3 $(top_srcdir)/src/create-catalog-v001.xml.py \ + _$@ \ + $(top_srcdir)/etc/domain_directories.tsv \ + $(top_srcdir)/etc/dependency_files.tsv \ + "$(top_srcdir)" \ + $(ttl_basenames) + mv _$@ $@ + check: \ - $(check_targets) + $(check_targets) \ + catalog-v001.xml # Reminder: diff exits non-0 on finding any differences. # Reminder: The $^ automatic Make variable is the name of all recipe prerequisites. @@ -51,4 +68,6 @@ check-%.ttl: \ || (echo "ERROR:src/review.mk:The local $< does not match the normalized version. If the above reported changes look fine, run 'cp .check-$< $<' while in the sub-folder ontology/$$(basename $< .ttl)/ to get a file ready to commit to Git." >&2 ; exit 1) clean: - @rm -f $(check_reference_basenames) + @rm -f \ + $(check_reference_basenames) \ + catalog-v001.xml diff --git a/tests/.gitignore b/tests/.gitignore index 4b1b6742..850ce0dd 100644 --- a/tests/.gitignore +++ b/tests/.gitignore @@ -1,6 +1,3 @@ .shapes.done.log -.venv.done.log -_* inheritance_review.ttl uco_monolithic.ttl -venv diff --git a/tests/Makefile b/tests/Makefile index aca7b3d8..e9c9b51f 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -15,8 +15,6 @@ SHELL := /bin/bash top_srcdir := $(shell cd .. ; pwd) -PYTHON3 ?= $(shell which python3.9 2>/dev/null || which python3.8 2>/dev/null || which python3.7 2>/dev/null || which python3.6 2>/dev/null || which python3) - uco_turtle_files := $(shell /bin/ls $(top_srcdir)/ontology/*/*.ttl $(top_srcdir)/ontology/*/*/*.ttl) imported_ontology_files := \ @@ -31,7 +29,7 @@ all: # Ensure the UCO-internal shapes pass their own tests. .shapes.done.log: \ $(top_srcdir)/ontology/owl/owl.ttl \ - .venv.done.log \ + $(top_srcdir)/.venv.done.log \ shapes/uco-closure-qc.ttl \ shapes/uco-qc.ttl $(MAKE) \ @@ -39,34 +37,11 @@ all: check touch $@ -# The two CASE-Utility... files are to trigger rebuilds based on command-line interface changes or version increments. -.venv.done.log: \ - $(top_srcdir)/.git_submodule_init.done.log \ - $(top_srcdir)/dependencies/CASE-Utility-SHACL-Inheritance-Reviewer/case_shacl_inheritance_reviewer/__init__.py \ - $(top_srcdir)/dependencies/CASE-Utility-SHACL-Inheritance-Reviewer/setup.cfg \ - requirements.txt - rm -rf venv - $(PYTHON3) -m venv \ - venv - source venv/bin/activate \ - && pip install \ - --upgrade \ - pip \ - setuptools \ - wheel - source venv/bin/activate \ - && pip install \ - $(top_srcdir)/dependencies/CASE-Utility-SHACL-Inheritance-Reviewer - source venv/bin/activate \ - && pip install \ - --requirement requirements.txt - touch $@ - check: \ check-dependencies \ inheritance_review.ttl \ uco_monolithic.ttl - source venv/bin/activate \ + source $(top_srcdir)/venv/bin/activate \ && pytest \ --ignore examples \ --ignore shapes \ @@ -76,7 +51,7 @@ check: \ check check-dependencies: \ - .venv.done.log + $(top_srcdir)/.venv.done.log $(MAKE) \ --directory dependencies \ check @@ -91,14 +66,12 @@ clean: @rm -f \ .*.done.log \ uco_monolithic.ttl - @rm -rf \ - venv inheritance_review.ttl: \ $(uco_turtle_files) \ - .venv.done.log + $(top_srcdir)/.venv.done.log rm -f _$@ - source venv/bin/activate \ + source $(top_srcdir)/venv/bin/activate \ && case_shacl_inheritance_reviewer \ --strict \ _$@ \ @@ -112,19 +85,19 @@ uco_monolithic.ttl: \ $(uco_turtle_files) \ .shapes.done.log \ thing.ttl - source venv/bin/activate \ + source $(top_srcdir)/venv/bin/activate \ && rdfpipe \ --input-format ttl \ --output-format ttl \ $(uco_turtle_files) \ > __$@ # Review UCO closure for versioning consistency. - source venv/bin/activate \ + source $(top_srcdir)/venv/bin/activate \ && rdfpipe \ __$@ \ $(imported_ontology_files) \ > ___$@ - source venv/bin/activate \ + source $(top_srcdir)/venv/bin/activate \ && pyshacl \ --data-file-format turtle \ --format turtle \ @@ -133,7 +106,7 @@ uco_monolithic.ttl: \ --shacl-file-format turtle \ ___$@ # Review UCO closure with SHACL-SHACL. - source venv/bin/activate \ + source $(top_srcdir)/venv/bin/activate \ && pyshacl \ --data-file-format turtle \ --format turtle \ @@ -145,7 +118,7 @@ uco_monolithic.ttl: \ # Closure tests have passed; remove closure file. rm ___$@ # Review UCO for practice conformance. - source venv/bin/activate \ + source $(top_srcdir)/venv/bin/activate \ && pyshacl \ --data-file-format turtle \ --format turtle \ @@ -154,7 +127,7 @@ uco_monolithic.ttl: \ --shacl-file-format turtle \ __$@ # Review UCO for OWL 2 DL conformance. - source venv/bin/activate \ + source $(top_srcdir)/venv/bin/activate \ && pyshacl \ --data-file-format turtle \ --format turtle \ diff --git a/tests/dependencies/Makefile b/tests/dependencies/Makefile index 2c066b86..7028d375 100644 --- a/tests/dependencies/Makefile +++ b/tests/dependencies/Makefile @@ -15,8 +15,6 @@ SHELL := /bin/bash top_srcdir := $(shell cd ../.. ; pwd) -tests_srcdir := $(top_srcdir)/tests - all: check: \ @@ -29,10 +27,10 @@ clean: uco-owl-co.done.log: \ $(top_srcdir)/dependencies/collections-ontology/collections.owl \ - $(tests_srcdir)/.venv.done.log \ + $(top_srcdir)/.venv.done.log \ $(top_srcdir)/ontology/owl/owl.ttl # Review for OWL 2 DL conformance. - source $(tests_srcdir)/venv/bin/activate \ + source $(top_srcdir)/venv/bin/activate \ && pyshacl \ --data-file-format xml \ --format turtle \ @@ -45,10 +43,10 @@ uco-owl-co.done.log: \ uco-owl-error.done.log: \ $(top_srcdir)/dependencies/error/docs/current/error.ttl \ - $(tests_srcdir)/.venv.done.log \ + $(top_srcdir)/.venv.done.log \ $(top_srcdir)/ontology/owl/owl.ttl # Review for OWL 2 DL conformance. - source $(tests_srcdir)/venv/bin/activate \ + source $(top_srcdir)/venv/bin/activate \ && pyshacl \ --data-file-format turtle \ --format turtle \ diff --git a/tests/examples/Makefile b/tests/examples/Makefile index ba966537..81c5dfad 100644 --- a/tests/examples/Makefile +++ b/tests/examples/Makefile @@ -35,6 +35,7 @@ all: \ location_XFAIL_validation.ttl \ message_thread_PASS_validation.ttl \ message_thread_XFAIL_validation.ttl \ + observable_creation_time_PASS_validation.ttl \ owl_axiom_PASS_validation.ttl \ owl_axiom_XFAIL_validation.ttl \ owl_properties_PASS_validation.ttl \ @@ -61,9 +62,9 @@ all: \ # characteristics. %_validation.ttl: \ %.json \ - $(tests_srcdir)/.venv.done.log \ + $(top_srcdir)/.venv.done.log \ $(tests_srcdir)/uco_monolithic.ttl - source $(tests_srcdir)/venv/bin/activate \ + source $(top_srcdir)/venv/bin/activate \ && pyshacl \ --allow-warnings \ --data-file-format json-ld \ @@ -103,6 +104,7 @@ check: \ location_XFAIL_validation.ttl \ message_thread_PASS_validation.ttl \ message_thread_XFAIL_validation.ttl \ + observable_creation_time_PASS_validation.ttl \ owl_axiom_PASS_validation.ttl \ owl_axiom_XFAIL_validation.ttl \ owl_properties_PASS_validation.ttl \ @@ -114,7 +116,7 @@ check: \ thread_PASS_validation.ttl \ thread_XFAIL_validation.ttl \ uco_thing_XFAIL_validation.ttl - source $(tests_srcdir)/venv/bin/activate \ + source $(top_srcdir)/venv/bin/activate \ && pytest \ --log-level=DEBUG diff --git a/tests/examples/hash_XFAIL_validation.ttl b/tests/examples/hash_XFAIL_validation.ttl index d520627c..e05552df 100644 --- a/tests/examples/hash_XFAIL_validation.ttl +++ b/tests/examples/hash_XFAIL_validation.ttl @@ -43,6 +43,10 @@ "SHA1"^^vocabulary:HashNameVocab "SHA224"^^vocabulary:HashNameVocab "SHA256"^^vocabulary:HashNameVocab + "SHA3-224"^^vocabulary:HashNameVocab + "SHA3-256"^^vocabulary:HashNameVocab + "SHA3-384"^^vocabulary:HashNameVocab + "SHA3-512"^^vocabulary:HashNameVocab "SHA384"^^vocabulary:HashNameVocab "SHA512"^^vocabulary:HashNameVocab "SSDEEP"^^vocabulary:HashNameVocab @@ -112,6 +116,10 @@ "SHA1"^^vocabulary:HashNameVocab "SHA224"^^vocabulary:HashNameVocab "SHA256"^^vocabulary:HashNameVocab + "SHA3-224"^^vocabulary:HashNameVocab + "SHA3-256"^^vocabulary:HashNameVocab + "SHA3-384"^^vocabulary:HashNameVocab + "SHA3-512"^^vocabulary:HashNameVocab "SHA384"^^vocabulary:HashNameVocab "SHA512"^^vocabulary:HashNameVocab "SSDEEP"^^vocabulary:HashNameVocab diff --git a/tests/examples/observable_creation_time_PASS.json b/tests/examples/observable_creation_time_PASS.json new file mode 100644 index 00000000..cde684fa --- /dev/null +++ b/tests/examples/observable_creation_time_PASS.json @@ -0,0 +1,24 @@ +{ + "@context": { + "core": "https://ontology.unifiedcyberontology.org/uco/core/", + "kb": "http://example.org/kb/", + "observable": "https://ontology.unifiedcyberontology.org/uco/observable/", + "rdfs": "http://www.w3.org/2000/01/rdf-schema#", + "xsd": "http://www.w3.org/2001/XMLSchema#" + }, + "@graph": [ + { + "@id": "kb:windows-thread-da52a01e-41cc-42d5-85be-ca14bfa10fd6", + "@type": "observable:WindowsThread", + "core:hasFacet": { + "@id": "kb:windows-thread-facet-4967ae35-f00b-49c8-9dd2-38e3bdf851e1", + "@type": "observable:WindowsThreadFacet", + "rdfs:comment": "In UCO 1.2.0, this will raise a warning. In UCO 2.0.0, this will raise an error.", + "observable:creationTime": { + "@type": "xsd:dateTime", + "@value": "2023-01-01T01:23:45Z" + } + } + } + ] +} diff --git a/tests/examples/observable_creation_time_PASS_validation.ttl b/tests/examples/observable_creation_time_PASS_validation.ttl new file mode 100644 index 00000000..60605c2c --- /dev/null +++ b/tests/examples/observable_creation_time_PASS_validation.ttl @@ -0,0 +1,27 @@ +@prefix observable: . +@prefix owl: . +@prefix rdf: . +@prefix rdfs: . +@prefix sh: . +@prefix xsd: . + +[] + a sh:ValidationReport ; + sh:conforms "true"^^xsd:boolean ; + sh:result [ + a sh:ValidationResult ; + sh:focusNode ; + sh:resultMessage "observable:creationTime is deprecated, and will be an error to use in UCO 2.0.0. observable:observableCreatedTime should be used instead."@en ; + sh:resultPath observable:creationTime ; + sh:resultSeverity sh:Warning ; + sh:sourceConstraintComponent sh:MaxCountConstraintComponent ; + sh:sourceShape [ + a sh:PropertyShape ; + sh:maxCount "0"^^xsd:integer ; + sh:message "observable:creationTime is deprecated, and will be an error to use in UCO 2.0.0. observable:observableCreatedTime should be used instead."@en ; + sh:path observable:creationTime ; + sh:severity sh:Warning ; + ] ; + ] ; + . + diff --git a/tests/examples/test_validation.py b/tests/examples/test_validation.py index d018b5ba..d0129801 100644 --- a/tests/examples/test_validation.py +++ b/tests/examples/test_validation.py @@ -365,6 +365,18 @@ def test_message_thread_PASS_validation() -> None: def test_message_thread_XFAIL_validation() -> None: confirm_validation_results("message_thread_XFAIL_validation.ttl", False) +def test_observable_creation_time_PASS() -> None: + confirm_validation_results( + "observable_creation_time_PASS_validation.ttl", + True, + expected_focus_node_severities={ + ( + "http://example.org/kb/windows-thread-facet-4967ae35-f00b-49c8-9dd2-38e3bdf851e1", + str(NS_SH.Warning) + ) + } + ) + def test_owl_axiom_PASS() -> None: confirm_validation_results( "owl_axiom_PASS_validation.ttl", diff --git a/tests/examples/uco_thing_XFAIL_validation.ttl b/tests/examples/uco_thing_XFAIL_validation.ttl index d126561a..ab5f6013 100644 --- a/tests/examples/uco_thing_XFAIL_validation.ttl +++ b/tests/examples/uco_thing_XFAIL_validation.ttl @@ -19,10 +19,11 @@ rdfs:seeAlso ; sh:message "UcoThings are suggested to end with a UUID."@en ; sh:select ''' - PREFIX uco-core: + PREFIX rdfs: + PREFIX core: SELECT $this WHERE { - $this a/rdfs:subClassOf* uco-core:UcoThing . + $this a/rdfs:subClassOf* core:UcoThing . FILTER ( ! REGEX ( STR($this), @@ -47,10 +48,11 @@ rdfs:seeAlso ; sh:message "UcoThings are suggested to end with a UUID."@en ; sh:select ''' - PREFIX uco-core: + PREFIX rdfs: + PREFIX core: SELECT $this WHERE { - $this a/rdfs:subClassOf* uco-core:UcoThing . + $this a/rdfs:subClassOf* core:UcoThing . FILTER ( ! REGEX ( STR($this), @@ -77,10 +79,11 @@ a sh:SPARQLConstraint ; sh:message "UcoThings are required to not be blank nodes."@en ; sh:select """ - PREFIX uco-core: + PREFIX rdfs: + PREFIX core: SELECT $this WHERE { - $this a/rdfs:subClassOf* uco-core:UcoThing . + $this a/rdfs:subClassOf* core:UcoThing . FILTER isBlank ($this) } """ ; @@ -105,10 +108,11 @@ rdfs:seeAlso ; sh:message "UcoThings are suggested to end with a UUID."@en ; sh:select ''' - PREFIX uco-core: + PREFIX rdfs: + PREFIX core: SELECT $this WHERE { - $this a/rdfs:subClassOf* uco-core:UcoThing . + $this a/rdfs:subClassOf* core:UcoThing . FILTER ( ! REGEX ( STR($this), diff --git a/tests/shapes/Makefile b/tests/shapes/Makefile index c53da2b5..3fbe92de 100644 --- a/tests/shapes/Makefile +++ b/tests/shapes/Makefile @@ -15,8 +15,6 @@ SHELL := /bin/bash top_srcdir := $(shell cd ../.. ; pwd) -tests_srcdir := $(top_srcdir)/tests - all: \ qc_monolithic.ttl $(MAKE) \ @@ -51,7 +49,7 @@ check: \ $(MAKE) \ --directory examples_uco_qc \ check - source $(tests_srcdir)/venv/bin/activate \ + source $(top_srcdir)/venv/bin/activate \ && pytest \ --log-level=DEBUG \ --verbose \ @@ -77,12 +75,12 @@ clean: qc_monolithic.ttl qc_monolithic.ttl: \ - $(tests_srcdir)/.venv.done.log \ + $(top_srcdir)/.venv.done.log \ $(top_srcdir)/.lib.done.log \ uco-closure-qc.ttl \ uco-qc.ttl rm -f __$@ _$@ - source $(tests_srcdir)/venv/bin/activate \ + source $(top_srcdir)/venv/bin/activate \ && rdfpipe \ --output-format turtle \ uco-closure-qc.ttl \ diff --git a/tests/shapes/examples_uco_owl/Makefile b/tests/shapes/examples_uco_owl/Makefile index 0b0f53ea..4f1a014e 100644 --- a/tests/shapes/examples_uco_owl/Makefile +++ b/tests/shapes/examples_uco_owl/Makefile @@ -15,9 +15,7 @@ SHELL := /bin/bash top_srcdir := $(shell cd ../../.. ; pwd) -tests_srcdir := $(top_srcdir)/tests - -shapes_srcdir := $(tests_srcdir)/shapes +shapes_srcdir := $(top_srcdir)/tests/shapes sample_ttls := $(wildcard *_PASS.ttl *_XFAIL.ttl) @@ -37,10 +35,10 @@ all: \ # characteristics. %_validation.ttl: \ %.ttl \ - $(tests_srcdir)/.venv.done.log \ + $(top_srcdir)/.venv.done.log \ $(top_srcdir)/ontology/owl/owl.ttl rm -f __$@ _$@ - source $(tests_srcdir)/venv/bin/activate \ + source $(top_srcdir)/venv/bin/activate \ && pyshacl \ --allow-warnings \ --data-file-format turtle \ diff --git a/tests/shapes/examples_uco_qc/Makefile b/tests/shapes/examples_uco_qc/Makefile index a04265f2..6e61d1c2 100644 --- a/tests/shapes/examples_uco_qc/Makefile +++ b/tests/shapes/examples_uco_qc/Makefile @@ -15,9 +15,7 @@ SHELL := /bin/bash top_srcdir := $(shell cd ../../.. ; pwd) -tests_srcdir := $(top_srcdir)/tests - -shapes_srcdir := $(tests_srcdir)/shapes +shapes_srcdir := $(top_srcdir)/tests/shapes sample_ttls := $(wildcard *_PASS.ttl *_XFAIL.ttl) @@ -37,10 +35,10 @@ all: \ # characteristics. %_validation.ttl: \ %.ttl \ - $(tests_srcdir)/.venv.done.log \ + $(top_srcdir)/.venv.done.log \ $(shapes_srcdir)/qc_monolithic.ttl rm -f __$@ _$@ - source $(tests_srcdir)/venv/bin/activate \ + source $(top_srcdir)/venv/bin/activate \ && pyshacl \ --allow-warnings \ --data-file-format turtle \ diff --git a/tests/shapes/examples_uco_qc/co_Collection_core_UcoObject_XFAIL.ttl b/tests/shapes/examples_uco_qc/co_Collection_core_UcoObject_XFAIL.ttl new file mode 100644 index 00000000..619d3675 --- /dev/null +++ b/tests/shapes/examples_uco_qc/co_Collection_core_UcoObject_XFAIL.ttl @@ -0,0 +1,57 @@ +@prefix co: . +@prefix core: . +@prefix ex: . +@prefix observable: . +@prefix owl: . +@prefix rdf: . +@prefix rdfs: . +@prefix xsd: . + +ex:IPAddressRoute + a owl:Class ; + rdfs:comment "This is expected to trigger a disjointed-classes violation, once subclass expansion occurs. This example copies subclass axioms from co:, core:, and observable:."@en ; + rdfs:subClassOf + co:List , + observable:ObservableObject + ; + . + +co:Bag + a owl:Class ; + rdfs:subClassOf co:Collection ; + . + +co:List + a owl:Class ; + rdfs:comment "This definition is excerpted from co:."@en ; + rdfs:subClassOf [ + a owl:Class ; + owl:intersectionOf ( + co:Bag + [ + a owl:Restriction ; + owl:onProperty co:item ; + owl:allValuesFrom co:ListItem ; + ] + ) ; + ] ; + . + +core:Item + a owl:Class ; + rdfs:subClassOf core:UcoObject ; + . + +observable:ObservableObject + a owl:Class ; + rdfs:subClassOf + core:Item , + observable:Observable + ; + . + +observable:Observable + a owl:Class ; + rdfs:subClassOf core:UcoObject ; + . + diff --git a/tests/shapes/examples_uco_qc/co_Collection_core_UcoObject_XFAIL_validation.ttl b/tests/shapes/examples_uco_qc/co_Collection_core_UcoObject_XFAIL_validation.ttl new file mode 100644 index 00000000..f8a4c0e8 --- /dev/null +++ b/tests/shapes/examples_uco_qc/co_Collection_core_UcoObject_XFAIL_validation.ttl @@ -0,0 +1,38 @@ +@prefix owl: . +@prefix rdf: . +@prefix rdfs: . +@prefix sh: . +@prefix uco-qc: . +@prefix xsd: . + +[] + a sh:ValidationReport ; + sh:conforms "false"^^xsd:boolean ; + sh:result [ + a sh:ValidationResult ; + sh:focusNode ; + sh:resultMessage "The focus node is defined as a subclass of two classes intended to be disjoint, co:Collection and core:UcoObject. See Issue 509 for suggested implementation alterations." ; + sh:resultSeverity sh:Violation ; + sh:sourceConstraint [ + a sh:SPARQLConstraint ; + rdfs:seeAlso ; + sh:message "The focus node is defined as a subclass of two classes intended to be disjoint, co:Collection and core:UcoObject. See Issue 509 for suggested implementation alterations."@en ; + sh:select """ + PREFIX co: + PREFIX core: + PREFIX rdfs: + SELECT $this + WHERE { + $this + rdfs:subClassOf*/(owl:intersectionOf/rdf:rest*/rdf:first)?/rdfs:subClassOf* co:Collection ; + rdfs:subClassOf* core:UcoObject ; + . + } + """ ; + ] ; + sh:sourceConstraintComponent sh:SPARQLConstraintComponent ; + sh:sourceShape uco-qc:disjointedness-co-collection-core-uco-object ; + sh:value ; + ] ; + . + diff --git a/tests/shapes/test_qc_shapes.py b/tests/shapes/test_qc_shapes.py index 0a59e6da..1d795e16 100644 --- a/tests/shapes/test_qc_shapes.py +++ b/tests/shapes/test_qc_shapes.py @@ -24,6 +24,16 @@ @pytest.mark.parametrize( ["filename", "expected_validation_result", "expected_focus_values"], [ + ( + "examples_uco_qc/co_Collection_core_UcoObject_XFAIL_validation.ttl", + False, + { + ( + URIRef("http://example.org/ontology/IPAddressRoute"), + URIRef("http://example.org/ontology/IPAddressRoute"), + ) + } + ), ( "examples_uco_owl/owl_incompatibleWith_shape_PASS_validation.ttl", True, diff --git a/tests/shapes/uco-qc.ttl b/tests/shapes/uco-qc.ttl index f3f247ba..e4e29db2 100644 --- a/tests/shapes/uco-qc.ttl +++ b/tests/shapes/uco-qc.ttl @@ -10,6 +10,28 @@ rdfs:comment "This ontology contains shapes meant to be used against the transitive closure of UCO, excluding imported, externally-developed ontologies."@en ; . +uco-qc:disjointedness-co-collection-core-uco-object + a sh:NodeShape ; + sh:sparql [ + a sh:SPARQLConstraint ; + rdfs:seeAlso ; + sh:message "The focus node is defined as a subclass of two classes intended to be disjoint, co:Collection and core:UcoObject. See Issue 509 for suggested implementation alterations."@en ; + sh:select """ + PREFIX co: + PREFIX core: + PREFIX rdfs: + SELECT $this + WHERE { + $this + rdfs:subClassOf*/(owl:intersectionOf/rdf:rest*/rdf:first)?/rdfs:subClassOf* co:Collection ; + rdfs:subClassOf* core:UcoObject ; + . + } + """ ; + ] ; + sh:targetClass owl:Class ; + . + uco-qc:owl-Ontology-shape a sh:NodeShape ; sh:property [ diff --git a/tests/test_uco_monolithic.py b/tests/test_uco_monolithic.py index 61bc6007..45d28d3b 100644 --- a/tests/test_uco_monolithic.py +++ b/tests/test_uco_monolithic.py @@ -220,16 +220,18 @@ def test_semi_open_vocabulary_owl_shacl_alignment(graph: Graph) -> None: shacl_list = rdf_list_to_member_list(graph, n_shacl_list) rdfs_list = rdf_list_to_member_list(graph, n_rdfs_list) - if rdfs_list == shacl_list: - logging.debug("Match") - else: - logging.debug(n_shacl_list) - logging.debug(n_rdfs_list) + if rdfs_list != shacl_list: shacl_tuple = tuple(shacl_list) rdfs_tuple = tuple(rdfs_list) computed.add((test_case[0], test_case[1], shacl_tuple, rdfs_tuple)) - assert expected == computed + try: + assert expected == computed + except AssertionError: + logging.error("Semi-open vocabulary lists are out of sync. See:") + for computed_result in computed: + logging.error("* %s", str(computed_result[0])) + raise def test_only_one_uco_class_is_owl_thing_direct_subclass(graph: Graph) -> None: