Merge pull request #1839 from 4dn-dcic/drs

Implement DRS for Fourfront

CHANGELOG.rst

@@ -6,9 +6,16 @@ fourfront
 Change Log
 ----------
 
+6.4.0
+=====
+
+* Implement and enable DRS API on File objects
+
+
 6.3.0
 =====
 
+
 `Node v18 Upgrade <https://github.com/4dn-dcic/fourfront/pull/1835>`_
 
 * Node in Docker make file and GA workflows migrated from v16 to v18

pyproject.toml

@@ -1,7 +1,7 @@
 [tool.poetry]
 # Note: Various modules refer to this system as "encoded", not "fourfront".
 name = "encoded"
-version = "6.3.0"
+version = "6.4.0"
 description = "4DN-DCIC Fourfront"
 authors = ["4DN-DCIC Team <[email protected]>"]
 license = "MIT"
@@ -47,7 +47,7 @@ colorama = "0.3.3"
 # we get odd 'pyo3_runtime.PanicException: Python API call failed' error on import
 # of cryptography.hazmat.bindings._rust in cryptography package. 2023-04-21.
 cryptography = "39.0.2"
-dcicsnovault = "^10.0.2"
+dcicsnovault = "^10.0.4"
 dcicutils = "^7.5.0"
 elasticsearch = "7.13.4"
 elasticsearch-dsl = "^7.0.0"  # TODO: port code from cgap-portal to get rid of uses

src/encoded/__init__.py

@@ -329,16 +329,9 @@ def include_snovault(config: Configurator) -> None:
     config.include('snovault.settings')
     config.include('snovault.server_defaults')
 
-#xyzzy - uncommented
- #  # make search available if ES is configured
- #  if config.registry.settings.get('elasticsearch.server'):
- #      config.include('snovault.search.search')
- #      config.include('snovault.search.compound_search')
-
     # configure redis server in production.ini
     if 'redis.server' in config.registry.settings:
         config.include('snovault.redis')
-#xyzzy - uncommented
 
     config.commit()
 

src/encoded/tests/test_file.py

@@ -380,6 +380,7 @@ def pairs_file_json(award, experiment, lab, file_formats, quality_metric_pairsqc
     }
     return item
 
+
 @pytest.fixture
 def mcool_file_json(award, experiment, lab, file_formats):
     item = {
@@ -392,6 +393,7 @@ def mcool_file_json(award, experiment, lab, file_formats):
     }
     return item
 
+
 @pytest.fixture
 def bedGraph_file_json(award, experiment, lab, file_formats):
     item = {

src/encoded/tests/test_file_drs.py

@@ -0,0 +1,76 @@
+import pytest
+
+
+pytestmark = [pytest.mark.setone, pytest.mark.working]
+
+
+DRS_PREFIX = f'/ga4gh/drs/v1/objects'
+
+
+@pytest.fixture
+def mcool_file_json(award, experiment, lab, file_formats):
+    """ Duplicating fixture since these live in another file that is not shared """
+    item = {
+        'award': award['@id'],
+        'lab': lab['@id'],
+        'file_format': file_formats.get('mcool').get('uuid'),
+        'md5sum': '00000000000000000000000000000000',
+        'content_md5sum': '00000000000000000000000000000000',
+        'filename': 'my.cool.mcool',
+        'status': 'uploaded',
+    }
+    return item
+
+
+@pytest.fixture
+def file(testapp, award, experiment, lab, file_formats):
+    """ Duplicating fixture since these live in another file that is not shared """
+    item = {
+        'award': award['@id'],
+        'lab': lab['@id'],
+        'file_format': file_formats.get('fastq').get('uuid'),
+        'md5sum': '00000000000000000000000000000000',
+        'content_md5sum': '00000000000000000000000000000000',
+        'filename': 'my.fastq.gz',
+        'status': 'uploaded',
+    }
+    res = testapp.post_json('/file_fastq', item)
+    return res.json['@graph'][0]
+
+
+def validate_drs_conversion(drs_obj, meta, uri=None):
+    """ Validates drs object structure against the metadata in the db """
+    assert drs_obj['id'] == meta['@id']
+    assert drs_obj['created_time'] == meta['date_created']
+    assert drs_obj['drs_id'] == meta['accession']
+    assert drs_obj['self_uri'] == f'drs://localhost:80{meta["@id"]}@@drs' if not uri else uri
+    assert drs_obj['version'] == meta['md5sum']
+    assert drs_obj['name'] == meta['filename']
+    assert drs_obj['aliases'] == [meta['uuid']]
+
+
+def test_processed_file_drs_view(testapp, mcool_file_json):
+    """ Tests that processed mcool gives a valid DRS response """
+    meta = testapp.post_json('/file_processed', mcool_file_json).json['@graph'][0]
+    drs_meta = testapp.get(meta['@id'] + '@@drs').json
+    validate_drs_conversion(drs_meta, meta)
+    drs_meta = testapp.get(f'{DRS_PREFIX}/{meta["uuid"]}').json
+    validate_drs_conversion(drs_meta, meta, uri=f'{DRS_PREFIX}/{meta["uuid"]}')
+
+
+def test_fastq_file_drs_view(testapp, file):
+    """ Tests that a fastq file has valid DRS response """
+    drs_meta = testapp.get(file['@id'] + '@@drs').json
+    validate_drs_conversion(drs_meta, file)
+    drs_meta = testapp.get(f'{DRS_PREFIX}/{file["uuid"]}').json
+    validate_drs_conversion(drs_meta, file, uri=f'{DRS_PREFIX}/{file["uuid"]}')
+
+
+def test_fastq_file_drs_access(testapp, file):
+    """ Tests that access URLs are retrieved successfully """
+    drs_meta = testapp.get(file['@id'] + '@@drs').json
+    drs_object_uri = drs_meta['drs_id']
+    drs_object_download = testapp.get(f'/ga4gh/drs/v1/objects/{drs_object_uri}/access/').json
+    assert drs_object_download == {
+        'url': f'https://localhost:80/{drs_object_uri}/@@download'
+    }

src/encoded/types/file.py

@@ -1664,6 +1664,71 @@ def download(context, request):
         raise HTTPTemporaryRedirect(location=location)
 
 
+def build_drs_object_from_props(drs_object_base, props):
+    """ Takes in base properties for a DRS object we expect on all items and expands them if corresponding
+        properties exist
+    """
+    # fields that match exactly in name and structure
+    for exact_field in [
+        'description',
+    ]:
+        if exact_field in props:
+            drs_object_base[exact_field] = props[exact_field]
+
+    # size is required by DRS so take it or default to 0
+    drs_object_base['size'] = props.get('file_size', 0)
+
+    # use system uuid as alias
+    drs_object_base['aliases'] = [props['uuid']]
+
+    # fields that are mapped to different names/structure
+    if 'content_md5sum' in props:
+        drs_object_base['checksums'] = [
+            {
+                'checksum': props['content_md5sum'],
+                'type': 'md5'
+            }
+        ]
+        # use md5sum as version
+        drs_object_base['version'] = props['content_md5sum']
+    if 'filename' in props:
+        drs_object_base['name'] = props['filename']
+    if 'last_modified' in props:
+        drs_object_base['updated_time'] = props['last_modified']['date_modified']
+    return drs_object_base
+
+
+@view_config(name='drs', context=File, request_method='GET',
+             permission='view', subpath_segments=[0, 1])
+def drs(context, request):
+    """ DRS object implementation for file. """
+    rendered_object = request.embed(str(context.uuid), '@@object', as_user=True)
+    accession = rendered_object['accession']
+    drs_object_base = {
+        'id': rendered_object['@id'],
+        'created_time': rendered_object['date_created'],
+        'drs_id': accession,
+        'self_uri': f'drs://{request.host}{request.path}',
+        'access_methods': [
+            {
+                # always prefer https
+                'access_url': {
+                    'url': f'https://{request.host}/{accession}/@@download'
+                },
+                'type': 'https'
+            },
+            {
+                # but provide http as well in case we are not on prod
+                'access_url': {
+                    'url': f'http://{request.host}/{accession}/@@download'
+                },
+                'type': 'http'
+            },
+        ]
+    }
+    return build_drs_object_from_props(drs_object_base, rendered_object)
+
+
 def get_file_experiment_type(request, context, properties):
     """
     Get the string experiment_type value given a File context and properties.