Merge fixes from 'usnistgov/integration' into main

python/nistoar/pdr/preserv/bagger/midas3.py

@@ -1140,7 +1140,7 @@ def finalize_version(self, update_reason=None):
         if self.datafiles is None:
             self.ensure_res_metadata()
         if not self.sip.nerd:
-           self.sip.nerd = bag.nerdm_record(True)
+           self.sip.nerd = self.bagbldr.bag.nerdm_record(True)
 
         # determine the type of update under way
         uptype = _NO_UPDATE
@@ -1162,28 +1162,31 @@ def finalize_version(self, update_reason=None):
             if prepr:
                 lastver = prepr.latest_version(["bag-cache", "bag-store", "repo"])
             if lastver == "0":
-                lastver = matched.group(1)
-
-            ver = [int(f) for f in lastver.split('.')]
-            for i in range(len(ver), 3):
-                ver.append(0)
-
-            if len(self.datafiles) > 0:
-                # there're data files waiting to be included; it's a data update
-                uptype = _DATA_UPDATE
-                ver[1] += 1
-                ver[2] = 0
-                umsg = "incrementing for updated data files"
-                self.log.debug("data files being updated: %s", self.datafiles.keys())
+                self.log.debug("finalizing version: dataset has never been published; setting version to 1.0.0")
+                self.sip.nerd['version'] = "1.0.0"
+
             else:
-                # otherwise, this is a metadata update, which increments the
-                # third field.
-                uptype = _MDATA_UPDATE
-                ver[2] += 1
-                umsg = "incrementing for updated metadata only"
-
-            self.sip.nerd['version'] = ".".join([str(v) for v in ver])
-            self.log.info("finalizing version: %s: %s", umsg, self.sip.nerd['version'])
+                # it's been published before under version=lastver
+                ver = [int(f) for f in lastver.split('.')]
+                for i in range(len(ver), 3):
+                    ver.append(0)
+
+                if len(self.datafiles) > 0:
+                    # there're data files waiting to be included; it's a data update
+                    uptype = _DATA_UPDATE
+                    ver[1] += 1
+                    ver[2] = 0
+                    umsg = "incrementing for updated data files"
+                    self.log.debug("data files being updated: %s", self.datafiles.keys())
+                else:
+                    # otherwise, this is a metadata update, which increments the
+                    # third field.
+                    uptype = _MDATA_UPDATE
+                    ver[2] += 1
+                    umsg = "incrementing for updated metadata only"
+
+                self.sip.nerd['version'] = ".".join([str(v) for v in ver])
+                self.log.info("finalizing version: %s: %s", umsg, self.sip.nerd['version'])
 
         else:
             self.log.info("finalizing version: sticking with explicitly set value: %s",

python/nistoar/pdr/publish/midas3/wsgi.py

@@ -414,6 +414,7 @@ def do_POST(self, path):
 
         try:
             if pod.get('accessLevel','public') == "non-public":
+                log.info("Received a non-public POD: scrubbing record from publishing system")
                 self._svc.delete(pod['identifier'])
                 return self.send_ok("Non-public POD Ignored", code=200)
 

python/nistoar/pdr/utils.py

@@ -141,6 +141,7 @@ def open(self, mode=None):
                     pass
             self._fo = None
             self._writing = None
+            raise
 
         if fcntl:
             lock_type = (self._writing and fcntl.LOCK_EX) or fcntl.LOCK_SH

python/tests/nistoar/pdr/preserv/bagger/test_midas3.py

@@ -853,6 +853,24 @@ def test_fileExaminer_autolaunch(self):
         self.assertIn('checksum', fmd)
 
     def test_finalize_version(self):
+        stagedir = os.path.join(self.bagparent,"stage")
+        if not os.path.isdir(stagedir):
+            os.mkdir(stagedir)
+        nerddir = os.path.join(stagedir, "_nerd")
+        if not os.path.isdir(nerddir):
+            os.mkdir(nerddir)
+        storedir = os.path.join(self.bagparent,"store")
+        if not os.path.isdir(storedir):
+            os.mkdir(storedir)
+        config = {
+            'repo_access': {
+                'headbag_cache': stagedir,
+                'store_dir': storedir,
+            }
+        }
+        self.bagr = midas.MIDASMetadataBagger.fromMIDAS(self.midasid, self.bagparent,
+                                                        self.revdir, config=config)
+
         # because there is data in the review directory, this will be seen
         # as a metadata update.
         inpodfile = os.path.join(self.revdir,"1491","_pod.json")
@@ -864,12 +882,25 @@ def test_finalize_version(self):
         self.bagr.sip.nerd = nerd
         self.assertEqual(nerd['version'], "1.0.0+ (in edit)")
 
+        nerd = self.bagr.finalize_version()
+        # it's never been published before, so version goes to 1.0.0
+        self.assertEqual(nerd['version'], "1.0.0")
+        self.assertEqual(self.bagr.sip.nerd['version'], "1.0.0")
+        nerd = self.bagr.bagbldr.bag.nerd_metadata_for('', True)
+        self.assertEqual(nerd['version'], "1.0.0")
+
+        # now pretend like it has.
+        self.bagr.bagbldr.update_annotations_for('', {'version': "1.0.0+ (in edit)"})
+        with open(os.path.join(stagedir, self.midasid+".1_0_0.mbag0_4-0.zip"), 'w') as fd:
+            fd.write("\n")
+        self.bagr = midas.MIDASMetadataBagger.fromMIDAS(self.midasid, self.bagparent,
+                                                        self.revdir, config=config)
         nerd = self.bagr.finalize_version()
         self.assertEqual(nerd['version'], "1.1.0")
-        self.assertEqual(self.bagr.sip.nerd['version'], "1.1.0")
         nerd = self.bagr.bagbldr.bag.nerd_metadata_for('', True)
         self.assertEqual(nerd['version'], "1.1.0")
 
+
 class TestMIDASMetadataBaggerUpload(test.TestCase):
 
     testsip = os.path.join(datadir, "midassip")
@@ -985,6 +1016,24 @@ def test_available_files(self):
         self.assertEqual(len(datafiles), 1)
 
     def test_finalize_version(self):
+        stagedir = os.path.join(self.bagparent,"stage")
+        if not os.path.isdir(stagedir):
+            os.mkdir(stagedir)
+        nerddir = os.path.join(stagedir, "_nerd")
+        if not os.path.isdir(nerddir):
+            os.mkdir(nerddir)
+        storedir = os.path.join(self.bagparent,"store")
+        if not os.path.isdir(storedir):
+            os.mkdir(storedir)
+        config = {
+            'repo_access': {
+                'headbag_cache': stagedir,
+                'store_dir': storedir,
+            }
+        }
+        self.bagr = midas.MIDASMetadataBagger.fromMIDAS(self.midasid, self.bagparent,
+                                                        None, self.upldir, config=config)
+
         inpodfile = os.path.join(self.upldir,"1491","_pod.json")
         self.bagr.apply_pod(inpodfile)
         self.bagr.ensure_data_files(examine="sync")
@@ -995,9 +1044,21 @@ def test_finalize_version(self):
         self.bagr.sip.nerd = nerd
         self.assertEqual(nerd['version'], "1.0.0+ (in edit)")
 
+        nerd = self.bagr.finalize_version()
+        self.assertEqual(nerd['version'], "1.0.0")
+        self.assertEqual(self.bagr.sip.nerd['version'], "1.0.0")
+        nerd = self.bagr.bagbldr.bag.nerd_metadata_for('', True)
+        self.assertEqual(nerd['version'], "1.0.0")
+
+        # now pretend like it has.
+        self.bagr.bagbldr.update_annotations_for('', {'version': "1.0.0+ (in edit)"})
+        with open(os.path.join(stagedir, self.midasid+".1_0_0.mbag0_4-0.zip"), 'w') as fd:
+            fd.write("\n")
+        self.bagr = midas.MIDASMetadataBagger.fromMIDAS(self.midasid, self.bagparent,
+                                                        None, self.upldir, config=config)
+        self.bagr.datafiles = {}
         nerd = self.bagr.finalize_version()
         self.assertEqual(nerd['version'], "1.0.1")
-        self.assertEqual(self.bagr.sip.nerd['version'], "1.0.1")
         nerd = self.bagr.bagbldr.bag.nerd_metadata_for('', True)
         self.assertEqual(nerd['version'], "1.0.1")