From 1a410cb98557db7ad60bd8cf1943db3ad6ea2b95 Mon Sep 17 00:00:00 2001 From: Ray Plante Date: Tue, 28 Jul 2020 18:05:29 -0400 Subject: [PATCH 1/2] pdr.utils bug fix: LockedFile: reraise on exception while opening --- python/nistoar/pdr/publish/midas3/wsgi.py | 1 + python/nistoar/pdr/utils.py | 1 + 2 files changed, 2 insertions(+) diff --git a/python/nistoar/pdr/publish/midas3/wsgi.py b/python/nistoar/pdr/publish/midas3/wsgi.py index 6019c2e9c..5a118bd49 100644 --- a/python/nistoar/pdr/publish/midas3/wsgi.py +++ b/python/nistoar/pdr/publish/midas3/wsgi.py @@ -414,6 +414,7 @@ def do_POST(self, path): try: if pod.get('accessLevel','public') == "non-public": + log.info("Received a non-public POD: scrubbing record from publishing system") self._svc.delete(pod['identifier']) return self.send_ok("Non-public POD Ignored", code=200) diff --git a/python/nistoar/pdr/utils.py b/python/nistoar/pdr/utils.py index 3f38ff115..93daaa1e9 100644 --- a/python/nistoar/pdr/utils.py +++ b/python/nistoar/pdr/utils.py @@ -141,6 +141,7 @@ def open(self, mode=None): pass self._fo = None self._writing = None + raise if fcntl: lock_type = (self._writing and fcntl.LOCK_EX) or fcntl.LOCK_SH From 89d094238c78a60d39fd6aaf0e5a6b2d422dcf16 Mon Sep 17 00:00:00 2001 From: Ray Plante Date: Tue, 28 Jul 2020 18:06:27 -0400 Subject: [PATCH 2/2] midas3: fix finalize_version(): set initial version if never published --- python/nistoar/pdr/preserv/bagger/midas3.py | 47 +++++++------- .../nistoar/pdr/preserv/bagger/test_midas3.py | 65 ++++++++++++++++++- 2 files changed, 88 insertions(+), 24 deletions(-) diff --git a/python/nistoar/pdr/preserv/bagger/midas3.py b/python/nistoar/pdr/preserv/bagger/midas3.py index dd19b9452..9cfa4ae83 100644 --- a/python/nistoar/pdr/preserv/bagger/midas3.py +++ b/python/nistoar/pdr/preserv/bagger/midas3.py @@ -1140,7 +1140,7 @@ def finalize_version(self, update_reason=None): if self.datafiles is None: self.ensure_res_metadata() if not self.sip.nerd: - self.sip.nerd = bag.nerdm_record(True) + self.sip.nerd = self.bagbldr.bag.nerdm_record(True) # determine the type of update under way uptype = _NO_UPDATE @@ -1162,28 +1162,31 @@ def finalize_version(self, update_reason=None): if prepr: lastver = prepr.latest_version(["bag-cache", "bag-store", "repo"]) if lastver == "0": - lastver = matched.group(1) - - ver = [int(f) for f in lastver.split('.')] - for i in range(len(ver), 3): - ver.append(0) - - if len(self.datafiles) > 0: - # there're data files waiting to be included; it's a data update - uptype = _DATA_UPDATE - ver[1] += 1 - ver[2] = 0 - umsg = "incrementing for updated data files" - self.log.debug("data files being updated: %s", self.datafiles.keys()) + self.log.debug("finalizing version: dataset has never been published; setting version to 1.0.0") + self.sip.nerd['version'] = "1.0.0" + else: - # otherwise, this is a metadata update, which increments the - # third field. - uptype = _MDATA_UPDATE - ver[2] += 1 - umsg = "incrementing for updated metadata only" - - self.sip.nerd['version'] = ".".join([str(v) for v in ver]) - self.log.info("finalizing version: %s: %s", umsg, self.sip.nerd['version']) + # it's been published before under version=lastver + ver = [int(f) for f in lastver.split('.')] + for i in range(len(ver), 3): + ver.append(0) + + if len(self.datafiles) > 0: + # there're data files waiting to be included; it's a data update + uptype = _DATA_UPDATE + ver[1] += 1 + ver[2] = 0 + umsg = "incrementing for updated data files" + self.log.debug("data files being updated: %s", self.datafiles.keys()) + else: + # otherwise, this is a metadata update, which increments the + # third field. + uptype = _MDATA_UPDATE + ver[2] += 1 + umsg = "incrementing for updated metadata only" + + self.sip.nerd['version'] = ".".join([str(v) for v in ver]) + self.log.info("finalizing version: %s: %s", umsg, self.sip.nerd['version']) else: self.log.info("finalizing version: sticking with explicitly set value: %s", diff --git a/python/tests/nistoar/pdr/preserv/bagger/test_midas3.py b/python/tests/nistoar/pdr/preserv/bagger/test_midas3.py index 1847bff25..0e0b8cf53 100644 --- a/python/tests/nistoar/pdr/preserv/bagger/test_midas3.py +++ b/python/tests/nistoar/pdr/preserv/bagger/test_midas3.py @@ -853,6 +853,24 @@ def test_fileExaminer_autolaunch(self): self.assertIn('checksum', fmd) def test_finalize_version(self): + stagedir = os.path.join(self.bagparent,"stage") + if not os.path.isdir(stagedir): + os.mkdir(stagedir) + nerddir = os.path.join(stagedir, "_nerd") + if not os.path.isdir(nerddir): + os.mkdir(nerddir) + storedir = os.path.join(self.bagparent,"store") + if not os.path.isdir(storedir): + os.mkdir(storedir) + config = { + 'repo_access': { + 'headbag_cache': stagedir, + 'store_dir': storedir, + } + } + self.bagr = midas.MIDASMetadataBagger.fromMIDAS(self.midasid, self.bagparent, + self.revdir, config=config) + # because there is data in the review directory, this will be seen # as a metadata update. inpodfile = os.path.join(self.revdir,"1491","_pod.json") @@ -864,12 +882,25 @@ def test_finalize_version(self): self.bagr.sip.nerd = nerd self.assertEqual(nerd['version'], "1.0.0+ (in edit)") + nerd = self.bagr.finalize_version() + # it's never been published before, so version goes to 1.0.0 + self.assertEqual(nerd['version'], "1.0.0") + self.assertEqual(self.bagr.sip.nerd['version'], "1.0.0") + nerd = self.bagr.bagbldr.bag.nerd_metadata_for('', True) + self.assertEqual(nerd['version'], "1.0.0") + + # now pretend like it has. + self.bagr.bagbldr.update_annotations_for('', {'version': "1.0.0+ (in edit)"}) + with open(os.path.join(stagedir, self.midasid+".1_0_0.mbag0_4-0.zip"), 'w') as fd: + fd.write("\n") + self.bagr = midas.MIDASMetadataBagger.fromMIDAS(self.midasid, self.bagparent, + self.revdir, config=config) nerd = self.bagr.finalize_version() self.assertEqual(nerd['version'], "1.1.0") - self.assertEqual(self.bagr.sip.nerd['version'], "1.1.0") nerd = self.bagr.bagbldr.bag.nerd_metadata_for('', True) self.assertEqual(nerd['version'], "1.1.0") + class TestMIDASMetadataBaggerUpload(test.TestCase): testsip = os.path.join(datadir, "midassip") @@ -985,6 +1016,24 @@ def test_available_files(self): self.assertEqual(len(datafiles), 1) def test_finalize_version(self): + stagedir = os.path.join(self.bagparent,"stage") + if not os.path.isdir(stagedir): + os.mkdir(stagedir) + nerddir = os.path.join(stagedir, "_nerd") + if not os.path.isdir(nerddir): + os.mkdir(nerddir) + storedir = os.path.join(self.bagparent,"store") + if not os.path.isdir(storedir): + os.mkdir(storedir) + config = { + 'repo_access': { + 'headbag_cache': stagedir, + 'store_dir': storedir, + } + } + self.bagr = midas.MIDASMetadataBagger.fromMIDAS(self.midasid, self.bagparent, + None, self.upldir, config=config) + inpodfile = os.path.join(self.upldir,"1491","_pod.json") self.bagr.apply_pod(inpodfile) self.bagr.ensure_data_files(examine="sync") @@ -995,9 +1044,21 @@ def test_finalize_version(self): self.bagr.sip.nerd = nerd self.assertEqual(nerd['version'], "1.0.0+ (in edit)") + nerd = self.bagr.finalize_version() + self.assertEqual(nerd['version'], "1.0.0") + self.assertEqual(self.bagr.sip.nerd['version'], "1.0.0") + nerd = self.bagr.bagbldr.bag.nerd_metadata_for('', True) + self.assertEqual(nerd['version'], "1.0.0") + + # now pretend like it has. + self.bagr.bagbldr.update_annotations_for('', {'version': "1.0.0+ (in edit)"}) + with open(os.path.join(stagedir, self.midasid+".1_0_0.mbag0_4-0.zip"), 'w') as fd: + fd.write("\n") + self.bagr = midas.MIDASMetadataBagger.fromMIDAS(self.midasid, self.bagparent, + None, self.upldir, config=config) + self.bagr.datafiles = {} nerd = self.bagr.finalize_version() self.assertEqual(nerd['version'], "1.0.1") - self.assertEqual(self.bagr.sip.nerd['version'], "1.0.1") nerd = self.bagr.bagbldr.bag.nerd_metadata_for('', True) self.assertEqual(nerd['version'], "1.0.1")