diff --git a/README.md b/README.md index 2838d9b05..1b2fbf4da 100644 --- a/README.md +++ b/README.md @@ -46,6 +46,10 @@ local artifact selection policy. This is used if the new fenwick policy excludes This is an implementation of the **file-validate** process that compares the inventory database against the back end storage at a storage site. +## vault +UNDER DEVELOPMENT: This is an implementation of an IVOA VOSpace +service that uses storage-inventory as the back end storage mechanism. + ## cadc-* These are libraries used in multiple services and applications. diff --git a/cadc-inventory-db/build.gradle b/cadc-inventory-db/build.gradle index f01493ba1..d96867abb 100644 --- a/cadc-inventory-db/build.gradle +++ b/cadc-inventory-db/build.gradle @@ -17,7 +17,7 @@ sourceCompatibility = 1.8 group = 'org.opencadc' -version = '0.14.6' +version = '0.15.0' description = 'OpenCADC Storage Inventory database library' def git_url = 'https://github.com/opencadc/storage-inventory' @@ -26,7 +26,9 @@ mainClassName = 'org.opencadc.inventory.db.version.Main' dependencies { compile 'org.opencadc:cadc-util:[1.9.5,2.0)' + compile 'org.opencadc:cadc-gms:[1.0.0,)' compile 'org.opencadc:cadc-inventory:[0.9.4,)' + compile 'org.opencadc:cadc-vos:[2.0,3.0)' testCompile 'junit:junit:[4.0,)' diff --git a/cadc-inventory-db/src/intTest/java/org/opencadc/inventory/db/TestUtil.java b/cadc-inventory-db/src/intTest/java/org/opencadc/inventory/db/TestUtil.java index 92ba24bcc..8ab42bf71 100644 --- a/cadc-inventory-db/src/intTest/java/org/opencadc/inventory/db/TestUtil.java +++ b/cadc-inventory-db/src/intTest/java/org/opencadc/inventory/db/TestUtil.java @@ -79,10 +79,11 @@ public class TestUtil { private static final Logger log = Logger.getLogger(TestUtil.class); - static String SERVER = "INVENTORY_TEST"; - static String DATABASE = "cadctest"; - static String SCHEMA = "inventory"; - static String TABLE_PREFIX = null; + public static String SERVER = "INVENTORY_TEST"; + public static String DATABASE = "cadctest"; + public static String SCHEMA = "inventory"; + public static String VOS_SCHEMA = "vospace"; + public static String TABLE_PREFIX = null; static { try { @@ -102,12 +103,17 @@ public class TestUtil { if (s != null) { SCHEMA = s.trim(); } + s = props.getProperty("vos_schema"); + if (s != null) { + VOS_SCHEMA = s.trim(); + } s = props.getProperty("tablePrefix"); if (s != null) { TABLE_PREFIX = s.trim(); } } - log.info("intTest database config: " + SERVER + " " + DATABASE + " " + SCHEMA + " " + TABLE_PREFIX); + log.info("intTest database config: " + SERVER + " " + DATABASE + " " + SCHEMA + " " + VOS_SCHEMA + + " tablePrefix=" + TABLE_PREFIX); } catch (Exception oops) { log.debug("failed to load/read optional db config", oops); } diff --git a/cadc-inventory-db/src/intTest/java/org/opencadc/vospace/db/NodeDAOTest.java b/cadc-inventory-db/src/intTest/java/org/opencadc/vospace/db/NodeDAOTest.java new file mode 100644 index 000000000..c24341c6e --- /dev/null +++ b/cadc-inventory-db/src/intTest/java/org/opencadc/vospace/db/NodeDAOTest.java @@ -0,0 +1,701 @@ +/* +************************************************************************ +******************* CANADIAN ASTRONOMY DATA CENTRE ******************* +************** CENTRE CANADIEN DE DONNÉES ASTRONOMIQUES ************** +* +* (c) 2023. (c) 2023. +* Government of Canada Gouvernement du Canada +* National Research Council Conseil national de recherches +* Ottawa, Canada, K1A 0R6 Ottawa, Canada, K1A 0R6 +* All rights reserved Tous droits réservés +* +* NRC disclaims any warranties, Le CNRC dénie toute garantie +* expressed, implied, or énoncée, implicite ou légale, +* statutory, of any kind with de quelque nature que ce +* respect to the software, soit, concernant le logiciel, +* including without limitation y compris sans restriction +* any warranty of merchantability toute garantie de valeur +* or fitness for a particular marchande ou de pertinence +* purpose. NRC shall not be pour un usage particulier. +* liable in any event for any Le CNRC ne pourra en aucun cas +* damages, whether direct or être tenu responsable de tout +* indirect, special or general, dommage, direct ou indirect, +* consequential or incidental, particulier ou général, +* arising from the use of the accessoire ou fortuit, résultant +* software. Neither the name de l'utilisation du logiciel. Ni +* of the National Research le nom du Conseil National de +* Council of Canada nor the Recherches du Canada ni les noms +* names of its contributors may de ses participants ne peuvent +* be used to endorse or promote être utilisés pour approuver ou +* products derived from this promouvoir les produits dérivés +* software without specific prior de ce logiciel sans autorisation +* written permission. préalable et particulière +* par écrit. +* +* This file is part of the Ce fichier fait partie du projet +* OpenCADC project. OpenCADC. +* +* OpenCADC is free software: OpenCADC est un logiciel libre ; +* you can redistribute it and/or vous pouvez le redistribuer ou le +* modify it under the terms of modifier suivant les termes de +* the GNU Affero General Public la “GNU Affero General Public +* License as published by the License” telle que publiée +* Free Software Foundation, par la Free Software Foundation +* either version 3 of the : soit la version 3 de cette +* License, or (at your option) licence, soit (à votre gré) +* any later version. toute version ultérieure. +* +* OpenCADC is distributed in the OpenCADC est distribué +* hope that it will be useful, dans l’espoir qu’il vous +* but WITHOUT ANY WARRANTY; sera utile, mais SANS AUCUNE +* without even the implied GARANTIE : sans même la garantie +* warranty of MERCHANTABILITY implicite de COMMERCIALISABILITÉ +* or FITNESS FOR A PARTICULAR ni d’ADÉQUATION À UN OBJECTIF +* PURPOSE. See the GNU Affero PARTICULIER. Consultez la Licence +* General Public License for Générale Publique GNU Affero +* more details. pour plus de détails. +* +* You should have received Vous devriez avoir reçu une +* a copy of the GNU Affero copie de la Licence Générale +* General Public License along Publique GNU Affero avec +* with OpenCADC. If not, see OpenCADC ; si ce n’est +* . pas le cas, consultez : +* . +* +************************************************************************ +*/ + +package org.opencadc.vospace.db; + +import ca.nrc.cadc.db.ConnectionConfig; +import ca.nrc.cadc.db.DBConfig; +import ca.nrc.cadc.db.DBUtil; +import ca.nrc.cadc.io.ResourceIterator; +import ca.nrc.cadc.util.Log4jInit; +import java.io.IOException; +import java.net.URI; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.sql.Connection; +import java.util.Map; +import java.util.TreeMap; +import java.util.UUID; +import javax.sql.DataSource; +import org.apache.log4j.Level; +import org.apache.log4j.Logger; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.opencadc.gms.GroupURI; +import org.opencadc.inventory.db.SQLGenerator; +import org.opencadc.inventory.db.TestUtil; +import org.opencadc.vospace.ContainerNode; +import org.opencadc.vospace.DataNode; +import org.opencadc.vospace.LinkNode; +import org.opencadc.vospace.Node; +import org.opencadc.vospace.NodeProperty; +import org.opencadc.vospace.VOS; + +/** + * + * @author pdowler + */ +public class NodeDAOTest { + private static final Logger log = Logger.getLogger(NodeDAOTest.class); + + static { + Log4jInit.setLevel("org.opencadc.inventory", Level.INFO); + Log4jInit.setLevel("org.opencadc.inventory.db", Level.INFO); + Log4jInit.setLevel("ca.nrc.cadc.db", Level.INFO); + Log4jInit.setLevel("org.opencadc.vospace", Level.INFO); + Log4jInit.setLevel("org.opencadc.vospace.db", Level.INFO); + } + + NodeDAO nodeDAO; + + public NodeDAOTest() throws Exception { + try { + DBConfig dbrc = new DBConfig(); + ConnectionConfig cc = dbrc.getConnectionConfig(TestUtil.SERVER, TestUtil.DATABASE); + DBUtil.PoolConfig pool = new DBUtil.PoolConfig(cc, 1, 6000L, "select 123"); + DBUtil.createJNDIDataSource("jdbc/NodeDAOTest", pool); + + Map config = new TreeMap<>(); + config.put(SQLGenerator.class.getName(), SQLGenerator.class); + config.put("jndiDataSourceName", "jdbc/NodeDAOTest"); + config.put("database", TestUtil.DATABASE); + config.put("schema", TestUtil.SCHEMA); + config.put("vosSchema", TestUtil.VOS_SCHEMA); + + this.nodeDAO = new NodeDAO(); + nodeDAO.setConfig(config); + + } catch (Exception ex) { + log.error("setup failed", ex); + throw ex; + } + } + + @Before + public void init_cleanup() throws Exception { + log.info("init database..."); + InitDatabaseVOS init = new InitDatabaseVOS(nodeDAO.getDataSource(), TestUtil.DATABASE, TestUtil.VOS_SCHEMA); + init.doInit(); + log.info("init database... OK"); + + log.info("clearing old content..."); + SQLGenerator gen = nodeDAO.getSQLGenerator(); + DataSource ds = nodeDAO.getDataSource(); + String sql = "delete from " + gen.getTable(ContainerNode.class); + log.info("pre-test cleanup: " + sql); + Connection con = ds.getConnection(); + con.createStatement().execute(sql); + con.close(); + log.info("clearing old content... OK"); + } + + @Test + public void testGetByID_NotFound() { + UUID id = UUID.randomUUID(); + Node a = nodeDAO.get(id); + Assert.assertNull(a); + } + + @Test + public void testGetByPath_NotFound() { + ContainerNode parent = new ContainerNode("not-found"); + Node a = nodeDAO.get(parent, "not-found"); + Assert.assertNull(a); + + UUID rootID = new UUID(0L, 0L); + ContainerNode root = new ContainerNode(rootID, "root"); + a = nodeDAO.get(root, "not-found"); + Assert.assertNull(a); + } + + @Test + public void testPutGetUpdateDeleteContainerNode() throws InterruptedException, + NoSuchAlgorithmException { + UUID rootID = new UUID(0L, 0L); + ContainerNode root = new ContainerNode(rootID, "root"); + + // put + ContainerNode orig = new ContainerNode("container-test"); + orig.parent = root; + orig.ownerID = "the-owner"; + nodeDAO.put(orig); + + // get-by-id + Node a = nodeDAO.get(orig.getID()); + Assert.assertNotNull(a); + log.info("found by id: " + a.getID() + " aka " + a); + Assert.assertEquals(orig.getID(), a.getID()); + Assert.assertEquals(orig.getName(), a.getName()); + Assert.assertEquals(root.getID(), a.parentID); + + // get-by-path + Node aa = nodeDAO.get(root, orig.getName()); + Assert.assertNotNull(aa); + log.info("found by path: " + aa.getID() + " aka " + aa); + Assert.assertEquals(orig.getID(), aa.getID()); + Assert.assertEquals(orig.getName(), aa.getName()); + Assert.assertEquals(root.getID(), a.parentID); + Assert.assertNotNull(aa.parentID); + Assert.assertEquals(root.getID(), aa.parentID); + + Assert.assertNull(a.parent); // get-node-by-id: comes pack without parent + Assert.assertEquals(orig.getName(), a.getName()); + Assert.assertEquals(orig.ownerID, a.ownerID); + Assert.assertEquals(orig.isPublic, a.isPublic); + Assert.assertEquals(orig.isLocked, a.isLocked); + Assert.assertEquals(orig.getReadOnlyGroup(), a.getReadOnlyGroup()); + Assert.assertEquals(orig.getReadWriteGroup(), a.getReadWriteGroup()); + Assert.assertEquals(orig.getProperties(), a.getProperties()); + + Assert.assertTrue(a instanceof ContainerNode); + ContainerNode c = (ContainerNode) a; + Assert.assertEquals(orig.inheritPermissions, c.inheritPermissions); + + // these are set in put + Assert.assertEquals(orig.getMetaChecksum(), a.getMetaChecksum()); + Assert.assertEquals(orig.getLastModified(), a.getLastModified()); + + URI mcs = a.computeMetaChecksum(MessageDigest.getInstance("MD5")); + Assert.assertEquals("metaChecksum", a.getMetaChecksum(), mcs); + + // update + Thread.sleep(10L); + orig.getReadOnlyGroup().add(new GroupURI(URI.create("ivo://opencadc.org/gms?g1"))); + orig.getReadWriteGroup().add(new GroupURI(URI.create("ivo://opencadc.org/gms?g3"))); + orig.getProperties().add(new NodeProperty(VOS.PROPERTY_URI_CONTENTLENGTH, "123")); + orig.isPublic = true; + orig.inheritPermissions = true; + nodeDAO.put(orig); + Node updated = nodeDAO.get(orig.getID()); + Assert.assertNotNull(updated); + Assert.assertEquals(orig.getID(), updated.getID()); + Assert.assertEquals(orig.getName(), updated.getName()); + Assert.assertTrue(a.getLastModified().before(updated.getLastModified())); + Assert.assertNotEquals(a.getMetaChecksum(), updated.getMetaChecksum()); + + Assert.assertNull(updated.parent); // get-node-by-id: comes pack without parent + Assert.assertEquals(orig.getName(), updated.getName()); + Assert.assertEquals(orig.ownerID, updated.ownerID); + Assert.assertEquals(orig.isPublic, updated.isPublic); + Assert.assertEquals(orig.isLocked, updated.isLocked); + Assert.assertEquals(orig.getReadOnlyGroup(), updated.getReadOnlyGroup()); + Assert.assertEquals(orig.getReadWriteGroup(), updated.getReadWriteGroup()); + Assert.assertEquals(orig.getProperties(), updated.getProperties()); + + Assert.assertTrue(updated instanceof ContainerNode); + ContainerNode uc = (ContainerNode) updated; + Assert.assertEquals(orig.inheritPermissions, uc.inheritPermissions); + + + nodeDAO.delete(orig.getID()); + Node gone = nodeDAO.get(orig.getID()); + Assert.assertNull(gone); + } + + @Test + public void testPutGetUpdateDeleteContainerNodeMax() throws InterruptedException, + NoSuchAlgorithmException { + UUID rootID = new UUID(0L, 0L); + ContainerNode root = new ContainerNode(rootID, "root"); + + // TODO: use get-by-path to find and remove the test node + + ContainerNode orig = new ContainerNode("container-test"); + orig.parent = root; + orig.ownerID = "the-owner"; + orig.isPublic = true; + orig.isLocked = false; + orig.inheritPermissions = false; + orig.getReadOnlyGroup().add(new GroupURI(URI.create("ivo://opencadc.org/gms?g1"))); + orig.getReadOnlyGroup().add(new GroupURI(URI.create("ivo://opencadc.org/gms?g2"))); + orig.getReadWriteGroup().add(new GroupURI(URI.create("ivo://opencadc.org/gms?g3"))); + orig.getReadWriteGroup().add(new GroupURI(URI.create("ivo://opencadc.org/gms?g6-g7"))); + orig.getReadWriteGroup().add(new GroupURI(URI.create("ivo://opencadc.org/gms?g6.g7"))); + orig.getReadWriteGroup().add(new GroupURI(URI.create("ivo://opencadc.org/gms?g6_g7"))); + orig.getReadWriteGroup().add(new GroupURI(URI.create("ivo://opencadc.org/gms?g6~g7"))); + + orig.getProperties().add(new NodeProperty(VOS.PROPERTY_URI_CONTENTLENGTH, "123")); + orig.getProperties().add(new NodeProperty(URI.create("custom:prop"), "spaces in value")); + orig.getProperties().add(new NodeProperty(URI.create("sketchy:a,b"), "comma in uri")); + orig.getProperties().add(new NodeProperty(URI.create("sketchy:funny"), "value-with-{delims}")); + nodeDAO.put(orig); + + // get-by-id + Node a = nodeDAO.get(orig.getID()); + Assert.assertNotNull(a); + log.info("found by id: " + a.getID() + " aka " + a); + Assert.assertEquals(orig.getID(), a.getID()); + Assert.assertEquals(orig.getName(), a.getName()); + Assert.assertNotNull(a.parentID); + Assert.assertEquals(root.getID(), a.parentID); + + // get-by-path + Node aa = nodeDAO.get(root, orig.getName()); + Assert.assertNotNull(aa); + log.info("found by path: " + aa.getID() + " aka " + aa); + Assert.assertEquals(orig.getID(), aa.getID()); + Assert.assertEquals(orig.getName(), aa.getName()); + Assert.assertNotNull(aa.parentID); + Assert.assertEquals(root.getID(), aa.parentID); + + Assert.assertNull(a.parent); // get-node-by-id: comes pack without parent + Assert.assertEquals(orig.getName(), a.getName()); + Assert.assertEquals(orig.ownerID, a.ownerID); + Assert.assertEquals(orig.isPublic, a.isPublic); + Assert.assertEquals(orig.isLocked, a.isLocked); + Assert.assertEquals(orig.getReadOnlyGroup(), a.getReadOnlyGroup()); + Assert.assertEquals(orig.getReadWriteGroup(), a.getReadWriteGroup()); + Assert.assertEquals(orig.getProperties(), a.getProperties()); + + Assert.assertTrue(a instanceof ContainerNode); + ContainerNode c = (ContainerNode) a; + Assert.assertEquals(orig.inheritPermissions, c.inheritPermissions); + + // these are set in put + Assert.assertEquals(orig.getMetaChecksum(), a.getMetaChecksum()); + Assert.assertEquals(orig.getLastModified(), a.getLastModified()); + + URI mcs = a.computeMetaChecksum(MessageDigest.getInstance("MD5")); + Assert.assertEquals("metaChecksum", a.getMetaChecksum(), mcs); + + // update + Thread.sleep(10L); + orig.isPublic = false; + orig.isLocked = true; + orig.getReadOnlyGroup().clear(); + orig.getReadOnlyGroup().add(new GroupURI(URI.create("ivo://opencadc.org/gms?g1"))); + orig.getReadWriteGroup().clear(); + orig.getReadWriteGroup().add(new GroupURI(URI.create("ivo://opencadc.org/gms?g3"))); + orig.getProperties().clear(); + orig.getProperties().add(new NodeProperty(VOS.PROPERTY_URI_CONTENTLENGTH, "123")); + orig.inheritPermissions = true; + nodeDAO.put(orig); + Node updated = nodeDAO.get(orig.getID()); + Assert.assertNotNull(updated); + Assert.assertEquals(orig.getID(), updated.getID()); + Assert.assertEquals(orig.getName(), updated.getName()); + Assert.assertTrue(a.getLastModified().before(updated.getLastModified())); + Assert.assertNotEquals(a.getMetaChecksum(), updated.getMetaChecksum()); + + Assert.assertNull(updated.parent); // get-node-by-id: comes pack without parent + Assert.assertEquals(orig.getName(), updated.getName()); + Assert.assertEquals(orig.ownerID, updated.ownerID); + Assert.assertEquals(orig.isPublic, updated.isPublic); + Assert.assertEquals(orig.isLocked, updated.isLocked); + Assert.assertEquals(orig.getReadOnlyGroup(), updated.getReadOnlyGroup()); + Assert.assertEquals(orig.getReadWriteGroup(), updated.getReadWriteGroup()); + Assert.assertEquals(orig.getProperties(), updated.getProperties()); + + Assert.assertTrue(updated instanceof ContainerNode); + ContainerNode uc = (ContainerNode) updated; + Assert.assertEquals(orig.inheritPermissions, uc.inheritPermissions); + + nodeDAO.delete(orig.getID()); + Node gone = nodeDAO.get(orig.getID()); + Assert.assertNull(gone); + } + + @Test + public void testPutGetUpdateDeleteDataNode() throws InterruptedException, + NoSuchAlgorithmException { + UUID rootID = new UUID(0L, 0L); + ContainerNode root = new ContainerNode(rootID, "root"); + + DataNode orig = new DataNode(UUID.randomUUID(), "data-test", URI.create("cadc:vault/" + UUID.randomUUID())); + orig.parent = root; + orig.ownerID = "the-owner"; + orig.isPublic = true; + orig.isLocked = false; + orig.getProperties().add(new NodeProperty(VOS.PROPERTY_URI_TYPE, "text/plain")); + orig.getProperties().add(new NodeProperty(VOS.PROPERTY_URI_DESCRIPTION, "this is the good stuff(tm)")); + nodeDAO.put(orig); + + // get-by-id + Node a = nodeDAO.get(orig.getID()); + Assert.assertNotNull(a); + log.info("found: " + a.getID() + " aka " + a); + Assert.assertEquals(orig.getID(), a.getID()); + Assert.assertEquals(orig.getName(), a.getName()); + Assert.assertEquals(root.getID(), a.parentID); + Assert.assertEquals(root.getID(), a.parentID); + + // get-by-path + Node aa = nodeDAO.get(root, orig.getName()); + Assert.assertNotNull(aa); + log.info("found: " + aa.getID() + " aka " + aa); + Assert.assertEquals(orig.getID(), aa.getID()); + Assert.assertEquals(orig.getName(), aa.getName()); + Assert.assertNotNull(aa.parentID); + Assert.assertEquals(root.getID(), aa.parentID); + + Assert.assertNull(a.parent); // get-node-by-id: comes pack without parent + Assert.assertEquals(orig.getName(), a.getName()); + Assert.assertEquals(orig.ownerID, a.ownerID); + Assert.assertEquals(orig.isPublic, a.isPublic); + Assert.assertEquals(orig.isLocked, a.isLocked); + Assert.assertEquals(orig.getReadOnlyGroup(), a.getReadOnlyGroup()); + Assert.assertEquals(orig.getReadWriteGroup(), a.getReadWriteGroup()); + Assert.assertEquals(orig.getProperties(), a.getProperties()); + + Assert.assertTrue(a instanceof DataNode); + DataNode dn = (DataNode) a; + Assert.assertEquals(orig.storageID, dn.storageID); + + // these are set in put + Assert.assertEquals(orig.getMetaChecksum(), a.getMetaChecksum()); + Assert.assertEquals(orig.getLastModified(), a.getLastModified()); + + URI mcs = a.computeMetaChecksum(MessageDigest.getInstance("MD5")); + Assert.assertEquals("metaChecksum", a.getMetaChecksum(), mcs); + + // update + Thread.sleep(10L); + orig.isPublic = false; + orig.isLocked = true; + orig.getReadOnlyGroup().clear(); + orig.getReadOnlyGroup().add(new GroupURI(URI.create("ivo://opencadc.org/gms?g1"))); + orig.getReadWriteGroup().clear(); + orig.getReadWriteGroup().add(new GroupURI(URI.create("ivo://opencadc.org/gms?g3"))); + orig.getProperties().clear(); + orig.getProperties().add(new NodeProperty(VOS.PROPERTY_URI_CONTENTLENGTH, "123")); + // don't change storageID + nodeDAO.put(orig); + Node updated = nodeDAO.get(orig.getID()); + Assert.assertNotNull(updated); + Assert.assertEquals(orig.getID(), updated.getID()); + Assert.assertEquals(orig.getName(), updated.getName()); + Assert.assertTrue(a.getLastModified().before(updated.getLastModified())); + Assert.assertNotEquals(a.getMetaChecksum(), updated.getMetaChecksum()); + + Assert.assertNull(updated.parent); // get-node-by-id: comes pack without parent + Assert.assertEquals(orig.getName(), updated.getName()); + Assert.assertEquals(orig.ownerID, updated.ownerID); + Assert.assertEquals(orig.isPublic, updated.isPublic); + Assert.assertEquals(orig.isLocked, updated.isLocked); + Assert.assertEquals(orig.getReadOnlyGroup(), updated.getReadOnlyGroup()); + Assert.assertEquals(orig.getReadWriteGroup(), updated.getReadWriteGroup()); + Assert.assertEquals(orig.getProperties(), updated.getProperties()); + + + Assert.assertTrue(a instanceof DataNode); + DataNode udn = (DataNode) updated; + Assert.assertEquals(orig.storageID, udn.storageID); + + nodeDAO.delete(orig.getID()); + Node gone = nodeDAO.get(orig.getID()); + Assert.assertNull(gone); + } + + @Test + public void testPutGetUpdateDeleteLinkNode() throws InterruptedException, + NoSuchAlgorithmException { + UUID rootID = new UUID(0L, 0L); + ContainerNode root = new ContainerNode(rootID, "root"); + + // TODO: use get-by-path to find and remove the test node + + LinkNode orig = new LinkNode("data-test", URI.create("vos://opencadc.org~srv/path/to/something")); + orig.parent = root; + orig.ownerID = "the-owner"; + orig.isPublic = true; + orig.isLocked = false; + orig.getProperties().add(new NodeProperty(VOS.PROPERTY_URI_DESCRIPTION, "link to the good stuff(tm)")); + nodeDAO.put(orig); + + // get-by-id + Node a = nodeDAO.get(orig.getID()); + Assert.assertNotNull(a); + log.info("found: " + a.getID() + " aka " + a); + Assert.assertEquals(orig.getID(), a.getID()); + Assert.assertEquals(orig.getName(), a.getName()); + Assert.assertEquals(root.getID(), a.parentID); + + // get-by-path + Node aa = nodeDAO.get(root, orig.getName()); + Assert.assertNotNull(aa); + log.info("found: " + aa.getID() + " aka " + aa); + Assert.assertEquals(orig.getID(), aa.getID()); + Assert.assertEquals(orig.getName(), aa.getName()); + Assert.assertNotNull(aa.parentID); + Assert.assertEquals(root.getID(), aa.parentID); + + Assert.assertNull(a.parent); // get-node-by-id: comes pack without parent + Assert.assertEquals(orig.getName(), a.getName()); + Assert.assertEquals(orig.ownerID, a.ownerID); + Assert.assertEquals(orig.isPublic, a.isPublic); + Assert.assertEquals(orig.isLocked, a.isLocked); + Assert.assertEquals(orig.getReadOnlyGroup(), a.getReadOnlyGroup()); + Assert.assertEquals(orig.getReadWriteGroup(), a.getReadWriteGroup()); + Assert.assertEquals(orig.getProperties(), a.getProperties()); + + Assert.assertTrue(a instanceof LinkNode); + LinkNode link = (LinkNode) a; + Assert.assertEquals(orig.getTarget(), link.getTarget()); + + // these are set in put + Assert.assertEquals(orig.getMetaChecksum(), a.getMetaChecksum()); + Assert.assertEquals(orig.getLastModified(), a.getLastModified()); + + URI mcs = a.computeMetaChecksum(MessageDigest.getInstance("MD5")); + Assert.assertEquals("metaChecksum", a.getMetaChecksum(), mcs); + + // update + Thread.sleep(10L); + orig.isPublic = false; + orig.isLocked = true; + orig.getReadOnlyGroup().clear(); + orig.getReadOnlyGroup().add(new GroupURI(URI.create("ivo://opencadc.org/gms?g1"))); + orig.getReadWriteGroup().clear(); + orig.getReadWriteGroup().add(new GroupURI(URI.create("ivo://opencadc.org/gms?g3"))); + orig.getProperties().clear(); + orig.getProperties().add(new NodeProperty(VOS.PROPERTY_URI_CONTENTLENGTH, "123")); + // don't change target + nodeDAO.put(orig); + Node updated = nodeDAO.get(orig.getID()); + Assert.assertNotNull(updated); + Assert.assertEquals(orig.getID(), updated.getID()); + Assert.assertEquals(orig.getName(), updated.getName()); + Assert.assertTrue(a.getLastModified().before(updated.getLastModified())); + Assert.assertNotEquals(a.getMetaChecksum(), updated.getMetaChecksum()); + + Assert.assertNull(updated.parent); // get-node-by-id: comes pack without parent + Assert.assertEquals(orig.getName(), updated.getName()); + Assert.assertEquals(orig.ownerID, updated.ownerID); + Assert.assertEquals(orig.isPublic, updated.isPublic); + Assert.assertEquals(orig.isLocked, updated.isLocked); + Assert.assertEquals(orig.getReadOnlyGroup(), updated.getReadOnlyGroup()); + Assert.assertEquals(orig.getReadWriteGroup(), updated.getReadWriteGroup()); + Assert.assertEquals(orig.getProperties(), updated.getProperties()); + + Assert.assertTrue(updated instanceof LinkNode); + LinkNode ulink = (LinkNode) updated; + Assert.assertEquals(orig.getTarget(), ulink.getTarget()); + + nodeDAO.delete(orig.getID()); + Node gone = nodeDAO.get(orig.getID()); + Assert.assertNull(gone); + } + + @Test + public void testGetWithLock() { + UUID rootID = new UUID(0L, 0L); + ContainerNode root = new ContainerNode(rootID, "root"); + + // put + ContainerNode orig = new ContainerNode("container-test"); + orig.parent = root; + orig.ownerID = "the-owner"; + nodeDAO.put(orig); + + // get-by-id + Node a = nodeDAO.get(orig.getID()); + Assert.assertNotNull(a); + log.info("found by id: " + a.getID() + " aka " + a); + Assert.assertEquals(orig.getID(), a.getID()); + Assert.assertEquals(orig.getName(), a.getName()); + Assert.assertEquals(root.getID(), a.parentID); + + // get with lock + Node locked = nodeDAO.lock(a); + Assert.assertNotNull(locked); + log.info("locked: " + a.getID() + " aka " + a); + + nodeDAO.delete(orig.getID()); + Node gone = nodeDAO.get(orig.getID()); + Assert.assertNull(gone); + } + + @Test + public void testContainerNodeIterator() throws IOException { + UUID rootID = new UUID(0L, 0L); + ContainerNode root = new ContainerNode(rootID, "root"); + + ContainerNode orig = new ContainerNode("container-test"); + orig.parent = root; + orig.ownerID = "the-owner"; + nodeDAO.put(orig); + + Node a = nodeDAO.get(orig.getID()); + Assert.assertNotNull(a); + log.info("found: " + a.getID() + " aka " + a); + Assert.assertEquals(orig.getID(), a.getID()); + Assert.assertEquals(orig.getName(), a.getName()); + + Assert.assertTrue(a instanceof ContainerNode); + ContainerNode cn = (ContainerNode) a; + Assert.assertTrue(nodeDAO.isEmpty(cn)); + + // these are set in put + Assert.assertEquals(orig.getMetaChecksum(), a.getMetaChecksum()); + Assert.assertEquals(orig.getLastModified(), a.getLastModified()); + try (ResourceIterator emptyIter = nodeDAO.iterator(orig, null, null)) { + Assert.assertNotNull(emptyIter); + Assert.assertFalse(emptyIter.hasNext()); + } // auto-close + + Node top = null; + try (ResourceIterator rootIter = nodeDAO.iterator(root, null, null)) { + if (rootIter.hasNext()) { + top = rootIter.next(); + } + } + Assert.assertNotNull(top); + Assert.assertEquals(orig.getID(), top.getID()); + + // add children + ContainerNode cont = new ContainerNode("container1"); + cont.parent = orig; + cont.ownerID = orig.ownerID; + DataNode data = new DataNode(UUID.randomUUID(), "data1", URI.create("cadc:vault/" + UUID.randomUUID())); + data.parent = orig; + data.ownerID = orig.ownerID; + LinkNode link = new LinkNode("link1", URI.create("cadc:ARCHIVE/data")); + link.parent = orig; + link.ownerID = orig.ownerID; + log.info("put child: " + cont + " of " + cont.parent); + nodeDAO.put(cont); + Assert.assertFalse(nodeDAO.isEmpty(cn)); + log.info("put child: " + data + " of " + data.parent); + nodeDAO.put(data); + Assert.assertFalse(nodeDAO.isEmpty(cn)); + log.info("put child: " + link + " of " + link.parent); + nodeDAO.put(link); + Assert.assertFalse(nodeDAO.isEmpty(cn)); + + Node c1; + Node c2; + Node c3; + try (ResourceIterator iter = nodeDAO.iterator(orig, null, null)) { + Assert.assertNotNull(iter); + Assert.assertTrue(iter.hasNext()); + c1 = iter.next(); + Assert.assertTrue(iter.hasNext()); + c2 = iter.next(); + Assert.assertTrue(iter.hasNext()); + c3 = iter.next(); + Assert.assertFalse(iter.hasNext()); + } + // default order: alpha + Assert.assertEquals(cont.getID(), c1.getID()); + Assert.assertEquals(cont.getName(), c1.getName()); + + Assert.assertEquals(data.getID(), c2.getID()); + Assert.assertEquals(data.getName(), c2.getName()); + + Assert.assertEquals(link.getID(), c3.getID()); + Assert.assertEquals(link.getName(), c3.getName()); + + // iterate with limit + try (ResourceIterator iter = nodeDAO.iterator(orig, 2, null)) { + Assert.assertNotNull(iter); + Assert.assertTrue(iter.hasNext()); + c1 = iter.next(); + Assert.assertTrue(iter.hasNext()); + c2 = iter.next(); + Assert.assertFalse(iter.hasNext()); + } + Assert.assertEquals(cont.getID(), c1.getID()); + Assert.assertEquals(cont.getName(), c1.getName()); + + Assert.assertEquals(data.getID(), c2.getID()); + Assert.assertEquals(data.getName(), c2.getName()); + + // iterate with start + try (ResourceIterator iter = nodeDAO.iterator(orig, null, c2.getName())) { + Assert.assertNotNull(iter); + Assert.assertTrue(iter.hasNext()); + c2 = iter.next(); + Assert.assertTrue(iter.hasNext()); + c3 = iter.next(); + Assert.assertFalse(iter.hasNext()); + } + Assert.assertEquals(data.getID(), c2.getID()); + Assert.assertEquals(data.getName(), c2.getName()); + + Assert.assertEquals(link.getID(), c3.getID()); + Assert.assertEquals(link.getName(), c3.getName()); + + // iterate with limit and start + try (ResourceIterator iter = nodeDAO.iterator(orig, 1, c2.getName())) { + Assert.assertNotNull(iter); + Assert.assertTrue(iter.hasNext()); + c2 = iter.next(); + Assert.assertFalse(iter.hasNext()); + } + Assert.assertEquals(data.getID(), c2.getID()); + Assert.assertEquals(data.getName(), c2.getName()); + + // depth first delete required but not enforced by DAO + nodeDAO.delete(cont.getID()); + nodeDAO.delete(data.getID()); + nodeDAO.delete(link.getID()); + nodeDAO.delete(orig.getID()); + Node gone = nodeDAO.get(orig.getID()); + Assert.assertNull(gone); + } +} diff --git a/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/AbstractDAO.java b/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/AbstractDAO.java index 7b823e304..237668ba9 100644 --- a/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/AbstractDAO.java +++ b/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/AbstractDAO.java @@ -86,8 +86,8 @@ import javax.naming.NamingException; import javax.sql.DataSource; import org.apache.log4j.Logger; -import org.opencadc.inventory.Entity; import org.opencadc.inventory.InventoryUtil; +import org.opencadc.persist.Entity; import org.springframework.jdbc.BadSqlGrammarException; import org.springframework.jdbc.core.JdbcTemplate; import org.springframework.jdbc.core.RowMapper; @@ -168,7 +168,7 @@ public DataSource getDataSource() { return dataSource; } - SQLGenerator getSQLGenerator() { + public SQLGenerator getSQLGenerator() { checkInit(); return gen; } @@ -191,6 +191,7 @@ public Map getParams() { ret.put("jndiDataSourceName", String.class); ret.put("database", String.class); ret.put("schema", String.class); + ret.put("vosSchema", String.class); // optional ret.put(SQLGenerator.class.getName(), Class.class); return ret; } @@ -224,9 +225,10 @@ public void setConfig(Map config) { String database = (String) config.get("database"); String schema = (String) config.get("schema"); + String vosSchema = (String) config.get("vosSchema"); try { - Constructor ctor = genClass.getConstructor(String.class, String.class); - this.gen = (SQLGenerator) ctor.newInstance(database, schema); + Constructor ctor = genClass.getConstructor(String.class, String.class, String.class); + this.gen = (SQLGenerator) ctor.newInstance(database, schema, vosSchema); } catch (Exception ex) { throw new RuntimeException("failed to instantiate SQLGenerator: " + genClass.getName(), ex); } diff --git a/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/EntityGet.java b/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/EntityGet.java index c065b58a6..029ca037b 100644 --- a/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/EntityGet.java +++ b/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/EntityGet.java @@ -68,7 +68,7 @@ package org.opencadc.inventory.db; import java.util.UUID; -import org.opencadc.inventory.Entity; +import org.opencadc.persist.Entity; import org.springframework.jdbc.core.JdbcTemplate; import org.springframework.jdbc.core.PreparedStatementCreator; diff --git a/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/EntityIteratorQuery.java b/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/EntityIteratorQuery.java index 782de3129..04e065bb8 100644 --- a/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/EntityIteratorQuery.java +++ b/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/EntityIteratorQuery.java @@ -67,7 +67,7 @@ package org.opencadc.inventory.db; -import java.util.Iterator; +import ca.nrc.cadc.io.ResourceIterator; import javax.sql.DataSource; /** @@ -76,5 +76,5 @@ * @param entity subclass */ public interface EntityIteratorQuery { - Iterator query(DataSource ds); + ResourceIterator query(DataSource ds); } diff --git a/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/EntityLock.java b/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/EntityLock.java index 238518d78..d99e41742 100644 --- a/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/EntityLock.java +++ b/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/EntityLock.java @@ -68,7 +68,7 @@ package org.opencadc.inventory.db; import java.util.UUID; -import org.opencadc.inventory.Entity; +import org.opencadc.persist.Entity; import org.springframework.jdbc.core.JdbcTemplate; import org.springframework.jdbc.core.PreparedStatementCreator; diff --git a/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/EntityPut.java b/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/EntityPut.java index 383347eba..e236c7a2b 100644 --- a/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/EntityPut.java +++ b/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/EntityPut.java @@ -69,7 +69,7 @@ package org.opencadc.inventory.db; -import org.opencadc.inventory.Entity; +import org.opencadc.persist.Entity; import org.springframework.jdbc.core.JdbcTemplate; import org.springframework.jdbc.core.PreparedStatementCreator; diff --git a/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/SQLGenerator.java b/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/SQLGenerator.java index fb5da9fec..6ac3dcbb0 100644 --- a/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/SQLGenerator.java +++ b/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/SQLGenerator.java @@ -3,7 +3,7 @@ ******************* CANADIAN ASTRONOMY DATA CENTRE ******************* ************** CENTRE CANADIEN DE DONNÉES ASTRONOMIQUES ************** * -* (c) 2022. (c) 2022. +* (c) 2023. (c) 2023. * Government of Canada Gouvernement du Canada * National Research Council Conseil national de recherches * Ottawa, Canada, K1A 0R6 Ottawa, Canada, K1A 0R6 @@ -89,18 +89,26 @@ import java.util.UUID; import javax.sql.DataSource; import org.apache.log4j.Logger; +import org.opencadc.gms.GroupURI; import org.opencadc.inventory.Artifact; import org.opencadc.inventory.DeletedArtifactEvent; import org.opencadc.inventory.DeletedStorageLocationEvent; -import org.opencadc.inventory.Entity; import org.opencadc.inventory.InventoryUtil; import org.opencadc.inventory.ObsoleteStorageLocation; import org.opencadc.inventory.SiteLocation; import org.opencadc.inventory.StorageLocation; import org.opencadc.inventory.StorageLocationEvent; import org.opencadc.inventory.StorageSite; +import org.opencadc.persist.Entity; +import org.opencadc.vospace.ContainerNode; +import org.opencadc.vospace.DataNode; +import org.opencadc.vospace.DeletedNodeEvent; +import org.opencadc.vospace.LinkNode; +import org.opencadc.vospace.Node; +import org.opencadc.vospace.NodeProperty; import org.springframework.dao.DataAccessException; import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.jdbc.core.PreparedStatementCreator; import org.springframework.jdbc.core.ResultSetExtractor; import org.springframework.jdbc.core.RowMapper; @@ -111,11 +119,12 @@ public class SQLGenerator { private static final Logger log = Logger.getLogger(SQLGenerator.class); - private final Map tableMap = new TreeMap(new ClassComp()); - private final Map columnMap = new TreeMap(new ClassComp()); + private final Map tableMap = new TreeMap<>(new ClassComp()); + private final Map columnMap = new TreeMap<>(new ClassComp()); protected final String database; // currently not used in SQL protected final String schema; // may be null + protected final String vosSchema; /** * Constructor. The database name is currently not used in any generated SQL; code assumes @@ -129,8 +138,13 @@ public class SQLGenerator { * @param schema schema name (may be null) */ public SQLGenerator(String database, String schema) { + this(database, schema, null); + } + + public SQLGenerator(String database, String schema, String vosSchema) { this.database = database; this.schema = schema; + this.vosSchema = vosSchema; init(); } @@ -203,6 +217,43 @@ protected void init() { "id" // last column is always PK }; this.columnMap.put(HarvestState.class, cols); + + // optional vospace + log.debug("vosSchema: " + vosSchema); + if (vosSchema != null) { + pref = vosSchema + "."; + tableMap.put(Node.class, pref + Node.class.getSimpleName()); + tableMap.put(DeletedNodeEvent.class, pref + DeletedNodeEvent.class.getSimpleName()); + + cols = new String[] { + "parentID", + "name", + "nodeType", + "ownerID", + "isPublic", + "isLocked", + "readOnlyGroups", + "readWriteGroups", + "properties", + "inheritPermissions", + "busy", + "storageID", + "target", + "lastModified", + "metaChecksum", + "id" // last column is always PK + }; + this.columnMap.put(Node.class, cols); + + cols = new String[] { + "nodeType", + "storageID", + "lastModified", + "metaChecksum", + "id" // last column is always PK + }; + this.columnMap.put(DeletedNodeEvent.class, cols); + } } private static class ClassComp implements Comparator { @@ -218,9 +269,28 @@ public String getCurrentTimeSQL() { return "SELECT now()"; } - // test usage - String getTable(Class c) { - return tableMap.get(c); + public String getTable(Class c) { + Class targetClass = c; + String ret = tableMap.get(targetClass); + if (ret == null) { + // enable finding a common table that stores subclass instances + targetClass = targetClass.getSuperclass(); + ret = tableMap.get(targetClass); + } + log.debug("table: " + c.getSimpleName() + " -> " + targetClass.getSimpleName() + " -> " + ret); + return ret; + } + + private String[] getColumns(Class c) { + Class targetClass = c; + String[] ret = columnMap.get(targetClass); + if (ret == null) { + // enable finding a common table that stores subclass instances + targetClass = targetClass.getSuperclass(); + ret = columnMap.get(targetClass); + } + log.debug("columns: " + c.getSimpleName() + " -> " + targetClass.getSimpleName() + " -> " + (ret == null ? null : ret.length)); + return ret; } public EntityGet getEntityGet(Class c) { @@ -235,10 +305,15 @@ public EntityGet getEntityGet(Class c, boolean forUpdate) { return new StorageSiteGet(forUpdate); } + if (Node.class.equals(c)) { + return new NodeGet(forUpdate); + } + if (forUpdate) { throw new UnsupportedOperationException("entity-get + forUpdate: " + c.getSimpleName()); } + // raw events are never locked for update if (DeletedArtifactEvent.class.equals(c)) { return new DeletedArtifactEventGet(); } @@ -251,17 +326,48 @@ public EntityGet getEntityGet(Class c, boolean forUpdate) { if (ObsoleteStorageLocation.class.equals(c)) { return new ObsoleteStorageLocationGet(); } + + if (DeletedNodeEvent.class.equals(c)) { + //return new DeletedNodeGet(); + } + if (HarvestState.class.equals(c)) { return new HarvestStateGet(); } + throw new UnsupportedOperationException("entity-get: " + c.getName()); } + public NodeCount getNodeCount() { + return new NodeCount(); + } + + public class NodeCount { + private UUID id; + + public void setID(UUID id) { + this.id = id; + } + + public int execute(JdbcTemplate jdbc) { + StringBuilder sb = new StringBuilder(); + sb.append("SELECT count(*) FROM ").append(getTable(Node.class)); + sb.append(" WHERE parentID = '").append(id.toString()).append("'"); + String sql = sb.toString(); + log.debug("NodeCount: " + sql); + int ret = jdbc.queryForObject(sql, Integer.class); + return ret; + } + } + public EntityIteratorQuery getEntityIteratorQuery(Class c) { if (Artifact.class.equals(c)) { return new ArtifactIteratorQuery(); } - throw new UnsupportedOperationException("entity-list: " + c.getName()); + if (Node.class.equals(c)) { + return new NodeIteratorQuery(); + } + throw new UnsupportedOperationException("entity-iterator: " + c.getName()); } public EntityList getEntityList(Class c) { @@ -271,13 +377,6 @@ public EntityList getEntityList(Class c) { throw new UnsupportedOperationException("entity-list: " + c.getName()); } - public EntityLock getEntityLock(Class c) { - if (Artifact.class.equals(c)) { - return new EntityLockImpl(c); - } - throw new UnsupportedOperationException("entity-list: " + c.getName()); - } - public EntityGet getSkeletonEntityGet(Class c) { EntityGet ret = new SkeletonGet(c); return ret; @@ -305,6 +404,12 @@ public EntityPut getEntityPut(Class c, boolean update) { if (HarvestState.class.equals(c)) { return new HarvestStatePut(update); } + if (Node.class.isAssignableFrom(c)) { + return new NodePut(update); + } + if (DeletedNodeEvent.class.equals(c)) { + //return new DeletedNodePut(update); + } throw new UnsupportedOperationException("entity-put: " + c.getName()); } @@ -312,41 +417,6 @@ public EntityDelete getEntityDelete(Class c) { return new EntityDeleteImpl(c); } - private class EntityLockImpl implements EntityLock { - private final Calendar utc = Calendar.getInstance(DateUtil.UTC); - private final Class entityClass; - private UUID id; - - EntityLockImpl(Class entityClass) { - this.entityClass = entityClass; - } - - @Override - public void setID(UUID id) { - this.id = id; - } - - @Override - public void execute(JdbcTemplate jdbc) throws EntityNotFoundException { - int n = jdbc.update(this); - if (n == 0) { - throw new EntityNotFoundException("not found: " + id); - } - } - - @Override - public PreparedStatement createPreparedStatement(Connection conn) throws SQLException { - String sql = getLockSQL(entityClass); - log.debug("EntityLockImpl: " + sql); - PreparedStatement prep = conn.prepareStatement(sql); - int col = 1; - prep.setObject(col++, id); - prep.setObject(col++, id); - - return prep; - } - } - private class SkeletonGet implements EntityGet { private UUID id; private final Class entityClass; @@ -428,7 +498,7 @@ public void setLocation(StorageLocation loc) { @Override public ObsoleteStorageLocation execute(JdbcTemplate jdbc) { - return (ObsoleteStorageLocation) jdbc.query(this, new ObsoleteStorageLocationExtractor()); + return jdbc.query(this, new ObsoleteStorageLocationExtractor()); } @Override @@ -443,7 +513,7 @@ public PreparedStatement createPreparedStatement(Connection conn) throws SQLExce String col = getKeyColumn(ObsoleteStorageLocation.class, true); sb.append(col).append(" = ?"); } else { - String[] cols = columnMap.get(ObsoleteStorageLocation.class); + String[] cols = getColumns(ObsoleteStorageLocation.class); sb.append(cols[0]).append(" = ?"); sb.append(" AND "); if (loc.storageBucket != null) { @@ -500,7 +570,7 @@ public PreparedStatement createPreparedStatement(Connection conn) throws SQLExce String col = getKeyColumn(HarvestState.class, true); sb.append(col).append(" = ?"); } else { - String[] cols = columnMap.get(HarvestState.class); + String[] cols = getColumns(HarvestState.class); sb.append(cols[0]).append(" = ?"); sb.append(" AND "); sb.append(cols[1]).append(" = ?"); @@ -704,7 +774,6 @@ public ResourceIterator query(DataSource ds) { private class StorageSiteGet implements EntityGet { private UUID id; - private URI uri; private final boolean forUpdate; public StorageSiteGet(boolean forUpdate) { @@ -731,6 +800,9 @@ public PreparedStatement createPreparedStatement(Connection conn) throws SQLExce } else { throw new IllegalStateException("primary key is null"); } + if (forUpdate) { + sb.append(" FOR UPDATE"); + } String sql = sb.toString(); log.debug("StorageSiteGet: " + sql); PreparedStatement prep = conn.prepareStatement(sql); @@ -759,6 +831,149 @@ public PreparedStatement createPreparedStatement(Connection conn) throws SQLExce } } + public class NodeGet implements EntityGet { + private UUID id; + private ContainerNode parent; + private String name; + private final boolean forUpdate; + + public NodeGet(boolean forUpdate) { + this.forUpdate = forUpdate; + } + + @Override + public void setID(UUID id) { + this.id = id; + } + + public void setPath(ContainerNode parent, String name) { + this.parent = parent; + this.name = name; + } + + @Override + public Node execute(JdbcTemplate jdbc) { + return (Node) jdbc.query(this, new NodeExtractor(parent)); + } + + @Override + public PreparedStatement createPreparedStatement(Connection conn) throws SQLException { + StringBuilder sb = getSelectFromSQL(Node.class, false); + sb.append(" WHERE "); + if (id != null) { + String col = getKeyColumn(Node.class, true); + sb.append(col).append(" = ?"); + } else if (parent != null && name != null) { + String pidCol = "parentID"; + String nameCol = "name"; + // TODO: better way to get column names? + sb.append(pidCol).append(" = ? and ").append(nameCol).append(" = ?"); + } else { + throw new IllegalStateException("primary key is null"); + } + if (forUpdate) { + sb.append(" FOR UPDATE"); + } + String sql = sb.toString(); + log.debug("Node: " + sql); + PreparedStatement prep = conn.prepareStatement(sql); + if (id != null) { + prep.setObject(1, id); + } else { + prep.setObject(1, parent.getID()); + prep.setObject(2, name); + } + + return prep; + } + } + + public class NodeIteratorQuery implements EntityIteratorQuery { + private ContainerNode parent; + private String start; + private Integer limit; + + public NodeIteratorQuery() { + } + + public void setParent(ContainerNode parent) { + this.parent = parent; + } + + public void setStart(String start) { + this.start = start; + } + + public void setLimit(Integer limit) { + this.limit = limit; + } + + @Override + public ResourceIterator query(DataSource ds) { + if (parent == null) { + throw new RuntimeException("BUG: cannot query for children with parent==null"); + } + + StringBuilder sb = getSelectFromSQL(Node.class, false); + sb.append(" WHERE parentID = ?"); + if (start != null) { + sb.append(" AND ? <= name"); + } + sb.append(" ORDER BY name ASC"); + if (limit != null) { + sb.append(" LIMIT ?"); + } + + String sql = sb.toString(); + log.debug("sql: " + sql); + + try { + Connection con = ds.getConnection(); + log.debug("NodeIteratorQuery: setAutoCommit(false)"); + con.setAutoCommit(false); + // defaults for options: ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_READ_ONLY + PreparedStatement ps = con.prepareStatement(sql); + ps.setFetchSize(1000); + ps.setFetchDirection(ResultSet.FETCH_FORWARD); + int col = 1; + + ps.setObject(col++, parent.getID()); + log.debug("parentID = " + parent.getID()); + if (start != null) { + ps.setString(col++, start); + log.debug("start = " + start); + } + if (limit != null) { + ps.setInt(col++, limit); + log.debug("limit = " + limit); + } + ResultSet rs = ps.executeQuery(); + + return new NodeResultSetIterator(con, rs, parent); + } catch (SQLException ex) { + throw new RuntimeException("BUG: artifact iterator query failed", ex); + } + + } + } + + private void safeSetBoolean(PreparedStatement prep, int col, Boolean value) throws SQLException { + log.debug("safeSetBoolean: " + col + " " + value); + if (value != null) { + prep.setBoolean(col, value); + } else { + prep.setNull(col, Types.BOOLEAN); + } + } + + private void safeSetString(PreparedStatement prep, int col, URI value) throws SQLException { + String v = null; + if (value != null) { + v = value.toASCIIString(); + } + safeSetString(prep, col, v); + } + private void safeSetString(PreparedStatement prep, int col, String value) throws SQLException { log.debug("safeSetString: " + col + " " + value); if (value != null) { @@ -786,6 +1001,24 @@ private void safeSetTimestamp(PreparedStatement prep, int col, Timestamp value, } } + private void safeSetArray(PreparedStatement prep, int col, Set values) throws SQLException { + + if (values != null && !values.isEmpty()) { + log.debug("safeSetArray: " + col + " " + values.size()); + String[] array1d = new String[values.size()]; + int i = 0; + for (GroupURI u : values) { + array1d[i] = u.getURI().toASCIIString(); + i++; + } + java.sql.Array arr = prep.getConnection().createArrayOf("text", array1d); + prep.setObject(col, arr); + } else { + log.debug("safeSetArray: " + col + " null"); + prep.setNull(col, Types.ARRAY); + } + } + private void safeSetArray(PreparedStatement prep, int col, UUID[] value) throws SQLException { if (value != null) { @@ -798,6 +1031,25 @@ private void safeSetArray(PreparedStatement prep, int col, UUID[] value) throws } } + private void safeSetProps(PreparedStatement prep, int col, Set values) throws SQLException { + + if (values != null && !values.isEmpty()) { + log.debug("safeSetProps: " + col + " " + values.size()); + String[][] array2d = new String[values.size()][2]; // TODO: w-h or h-w?? + int i = 0; + for (NodeProperty np : values) { + array2d[i][0] = np.getKey().toASCIIString(); + array2d[i][1] = np.getValue(); + i++; + } + java.sql.Array arr = prep.getConnection().createArrayOf("text", array2d); + prep.setObject(col, arr); + } else { + log.debug("safeSetProps: " + col + " = null"); + prep.setNull(col, Types.ARRAY); + } + } + private class ArtifactPut implements EntityPut { private final Calendar utc = Calendar.getInstance(DateUtil.UTC); private final boolean update; @@ -852,8 +1104,8 @@ public PreparedStatement createPreparedStatement(Connection conn) throws SQLExce safeSetString(prep, col++, value.storageLocation.getStorageID().toASCIIString()); safeSetString(prep, col++, value.storageLocation.storageBucket); } else { - safeSetString(prep, col++, null); // storageLocation.storageID - safeSetString(prep, col++, null); // storageLocation.storageBucket + safeSetString(prep, col++, (URI) null); // storageLocation.storageID + safeSetString(prep, col++, (URI) null); // storageLocation.storageBucket } safeSetTimestamp(prep, col++, new Timestamp(value.getLastModified().getTime()), utc); @@ -948,7 +1200,7 @@ public PreparedStatement createPreparedStatement(Connection conn) throws SQLExce if (value.getLocation().storageBucket != null) { safeSetString(prep, col++, value.getLocation().storageBucket); } else { - safeSetString(prep, col++, null); + safeSetString(prep, col++, (String) null); } prep.setTimestamp(col++, new Timestamp(value.getLastModified().getTime()), utc); @@ -1014,6 +1266,85 @@ public PreparedStatement createPreparedStatement(Connection conn) throws SQLExce } + private class NodePut implements EntityPut { + private final Calendar utc = Calendar.getInstance(DateUtil.UTC); + private final boolean update; + private Node value; + + NodePut(boolean update) { + this.update = update; + } + + @Override + public void setValue(Node value) { + this.value = value; + } + + @Override + public void execute(JdbcTemplate jdbc) { + jdbc.update(this); + } + + @Override + public PreparedStatement createPreparedStatement(Connection conn) throws SQLException { + String sql = null; + if (update) { + sql = getUpdateSQL(Node.class); + + } else { + sql = getInsertSQL(Node.class); + } + log.debug("NodePut: " + sql); + PreparedStatement prep = conn.prepareStatement(sql); + int col = 1; + + if (value.parentID == null) { + throw new RuntimeException("BUG: cannot put Node without a parentID: " + value); + } + prep.setObject(col++, value.parentID); + prep.setString(col++, value.getName()); + prep.setString(col++, value.getClass().getSimpleName().substring(0, 1)); // HACK + if (value.ownerID == null) { + throw new RuntimeException("BUG: cannot put Node without an ownerID: " + value); + } + prep.setString(col++, value.ownerID.toString()); + safeSetBoolean(prep, col++, value.isPublic); + safeSetBoolean(prep, col++, value.isLocked); + safeSetArray(prep, col++, value.getReadOnlyGroup()); + safeSetArray(prep, col++, value.getReadWriteGroup()); + safeSetProps(prep, col++, value.getProperties()); + if (value instanceof ContainerNode) { + ContainerNode cn = (ContainerNode) value; + safeSetBoolean(prep, col++, cn.inheritPermissions); + } else { + safeSetBoolean(prep, col++, null); + } + if (value instanceof DataNode) { + DataNode dn = (DataNode) value; + if (dn.storageID == null) { + throw new RuntimeException("BUG: cannot put DataNode without a storageID: " + value); + } + safeSetBoolean(prep, col++, dn.busy); + safeSetString(prep, col++, dn.storageID); + } else { + safeSetBoolean(prep, col++, null); + safeSetString(prep, col++, (URI) null); + } + if (value instanceof LinkNode) { + LinkNode ln = (LinkNode) value; + prep.setString(col++, ln.getTarget().toASCIIString()); + } else { + safeSetString(prep, col++, (URI) null); + } + + prep.setTimestamp(col++, new Timestamp(value.getLastModified().getTime()), utc); + prep.setString(col++, value.getMetaChecksum().toASCIIString()); + prep.setObject(col++, value.getID()); + + return prep; + } + } + private class EntityEventPut implements EntityPut { private final Calendar utc = Calendar.getInstance(DateUtil.UTC); private final boolean update; @@ -1083,11 +1414,13 @@ public PreparedStatement createPreparedStatement(Connection conn) throws SQLExce } private StringBuilder getSelectFromSQL(Class c, boolean entityCols) { - String tab = tableMap.get(c); - String[] cols = columnMap.get(c); + String tab = getTable(c); + Class targetClass = c; if (entityCols) { - cols = columnMap.get(Entity.class); + targetClass = Entity.class; } + String[] cols = getColumns(targetClass); + if (tab == null || cols == null) { throw new IllegalArgumentException("BUG: no table/columns for class " + c.getName()); } @@ -1106,21 +1439,13 @@ private StringBuilder getSelectFromSQL(Class c, boolean entityCols) { return sb; } - private String getLockSQL(Class c) { - StringBuilder sb = new StringBuilder(); - String pk = getKeyColumn(c, true); - sb.append("UPDATE "); - sb.append(tableMap.get(c)); - sb.append(" SET ").append(pk).append(" = ? WHERE ").append(pk).append(" = ?"); - return sb.toString(); - } - private String getUpdateSQL(Class c) { StringBuilder sb = new StringBuilder(); + String tab = getTable(c); sb.append("UPDATE "); - sb.append(tableMap.get(c)); + sb.append(tab); sb.append(" SET "); - String[] cols = columnMap.get(c); + String[] cols = getColumns(c); for (int i = 0; i < cols.length - 1; i++) { // PK is last if (i > 0) { sb.append(","); @@ -1137,10 +1462,11 @@ private String getUpdateSQL(Class c) { private String getInsertSQL(Class c) { StringBuilder sb = new StringBuilder(); + String tab = getTable(c); sb.append("INSERT INTO "); - sb.append(tableMap.get(c)); + sb.append(tab); sb.append(" ("); - String[] cols = columnMap.get(c); + String[] cols = getColumns(c); for (int i = 0; i < cols.length; i++) { if (i > 0) { sb.append(","); @@ -1161,14 +1487,15 @@ private String getInsertSQL(Class c) { private String getDeleteSQL(Class c) { StringBuilder sb = new StringBuilder(); + String tab = getTable(c); sb.append("DELETE FROM "); - sb.append(tableMap.get(c)); + sb.append(tab); sb.append(" WHERE id = ?"); return sb.toString(); } private String getKeyColumn(Class c, boolean pk) { - String[] cols = columnMap.get(c); + String[] cols = getColumns(c); if (cols == null) { throw new IllegalArgumentException("BUG: no table/columns for class " + c.getName()); } @@ -1226,6 +1553,7 @@ private class ArtifactResultSetIterator implements ResourceIterator { private final Connection con; private final ResultSet rs; boolean hasRow; + boolean closeWhenDone = false; // not a pooled connection ArtifactResultSetIterator(Connection con, ResultSet rs) throws SQLException { this.con = con; @@ -1234,7 +1562,14 @@ private class ArtifactResultSetIterator implements ResourceIterator { log.debug("ArtifactResultSetIterator: " + super.toString() + " ctor " + hasRow); if (!hasRow) { log.debug("ArtifactResultSetIterator: " + super.toString() + " ctor - setAutoCommit(true)"); - con.setAutoCommit(true); + try { + con.setAutoCommit(true); // commit txn + if (closeWhenDone) { + con.close(); // return to pool + } + } catch (SQLException unexpected) { + log.error("Connection.setAutoCommit(true) & close() failed", unexpected); + } } } @@ -1243,10 +1578,13 @@ public void close() throws IOException { if (hasRow) { log.debug("ArtifactResultSetIterator: " + super.toString() + " ctor - setAutoCommit(true)"); try { - con.setAutoCommit(true); - hasRow = false; - } catch (SQLException ex) { - throw new RuntimeException("BUG: artifact list query failed during close()", ex); + con.setAutoCommit(true); // commit txn + if (closeWhenDone) { + con.close(); // return to pool + } + hasRow = false; + } catch (SQLException unexpected) { + log.error("Connection.setAutoCommit(true) & close() failed", unexpected); } } } @@ -1263,17 +1601,27 @@ public Artifact next() { hasRow = rs.next(); if (!hasRow) { log.debug("ArtifactResultSetIterator: " + super.toString() + " DONE - setAutoCommit(true)"); - con.setAutoCommit(true); + try { + con.setAutoCommit(true); // commit txn + if (closeWhenDone) { + con.close(); // return to pool + } + } catch (SQLException unexpected) { + log.error("Connection.setAutoCommit(true) & close() failed", unexpected); + } } return ret; } catch (Exception ex) { if (hasRow) { log.debug("ArtifactResultSetIterator: " + super.toString() + " ResultSet.next() FAILED - setAutoCommit(true)"); try { - close(); + con.setAutoCommit(true); // commit txn + if (closeWhenDone) { + con.close(); // return to pool + } hasRow = false; - } catch (IOException unexpected) { - log.debug("BUG: unexpected IOException from close", unexpected); + } catch (SQLException unexpected) { + log.error("Connection.setAutoCommit(true) & close() failed", unexpected); } } throw new RuntimeException("BUG: artifact list query failed while iterating", ex); @@ -1281,6 +1629,83 @@ public Artifact next() { } } + private class NodeResultSetIterator implements ResourceIterator { + final Calendar utc = Calendar.getInstance(DateUtil.UTC); + private final Connection con; + private final ResultSet rs; + boolean hasRow; + + ContainerNode parent; + + public NodeResultSetIterator(Connection con, ResultSet rs, ContainerNode parent) throws SQLException { + this.con = con; + this.rs = rs; + this.parent = parent; + hasRow = rs.next(); + log.debug("NodeResultSetIterator: " + super.toString() + " ctor " + hasRow); + if (!hasRow) { + log.debug("NodeResultSetIterator: " + super.toString() + " ctor - setAutoCommit(true)"); + + try { + con.setAutoCommit(true); // commit txn + con.close(); // return to pool + } catch (SQLException ignore) { + log.error("Connection.setAutoCommit(true) & close() failed", ignore); + } + } + } + + @Override + public void close() throws IOException { + if (hasRow) { + log.debug("NodeResultSetIterator: " + super.toString() + " close - setAutoCommit(true)"); + try { + con.setAutoCommit(true); // commit txn + con.close(); // return to pool + hasRow = false; + } catch (SQLException ignore) { + log.error("Connection.setAutoCommit(true) & close() failed", ignore); + } + } + } + + @Override + public boolean hasNext() { + return hasRow; + } + + @Override + public Node next() { + try { + Node ret = mapRowToNode(rs, utc, parent); + hasRow = rs.next(); + if (!hasRow) { + log.debug("NodeResultSetIterator: " + super.toString() + " DONE - setAutoCommit(true)"); + try { + con.setAutoCommit(true); // commit txn + con.close(); // return to pool + hasRow = false; + } catch (SQLException ignore) { + log.error("Connection.setAutoCommit(true) & close() failed", ignore); + } + } + return ret; + } catch (Exception ex) { + if (hasRow) { + log.debug("NodeResultSetIterator: " + super.toString() + " ResultSet.next() FAILED - setAutoCommit(true)"); + try { + con.setAutoCommit(true); // commit txn + con.close(); // return to pool + hasRow = false; + } catch (SQLException ignore) { + log.error("Connection.setAutoCommit(true) & close() failed", ignore); + } + } + throw new RuntimeException("BUG: node list query failed while iterating", ex); + } + } + } + private Artifact mapRowToArtifact(ResultSet rs, Calendar utc) throws SQLException { int col = 1; final URI uri = Util.getURI(rs, col++); @@ -1314,12 +1739,64 @@ private Artifact mapRowToArtifact(ResultSet rs, Calendar utc) throws SQLExceptio return a; } - private class ObsoleteStorageLocationExtractor implements ResultSetExtractor { + private Node mapRowToNode(ResultSet rs, Calendar utc, ContainerNode parent) throws SQLException { + int col = 1; + final UUID parentID = Util.getUUID(rs, col++); + final String name = rs.getString(col++); + final String nodeType = rs.getString(col++); + final String ownerID = rs.getString(col++); + final Boolean isPublic = Util.getBoolean(rs, col++); + final Boolean isLocked = Util.getBoolean(rs, col++); + final String rawROG = rs.getString(col++); + final String rawRWG = rs.getString(col++); + final String rawProps = rs.getString(col++); + final Boolean inheritPermissions = Util.getBoolean(rs, col++); + final Boolean busy = Util.getBoolean(rs, col++); + final URI storageID = Util.getURI(rs, col++); + final URI linkTarget = Util.getURI(rs, col++); + final Date lastModified = Util.getDate(rs, col++, utc); + final URI metaChecksum = Util.getURI(rs, col++); + final UUID id = Util.getUUID(rs, col++); + + Node ret; + if (nodeType.equals("C")) { + ContainerNode cn = new ContainerNode(id, name); + cn.inheritPermissions = inheritPermissions; + ret = cn; + } else if (nodeType.equals("D")) { + ret = new DataNode(id, name, storageID); + } else if (nodeType.equals("L")) { + ret = new LinkNode(id, name, linkTarget); + } else { + throw new RuntimeException("BUG: unexpected node type code: " + nodeType); + } + ret.parentID = parentID; + ret.ownerID = ownerID; + ret.isPublic = isPublic; + ret.isLocked = isLocked; + + if (rawROG != null) { + Util.parseArrayGroupURI(rawROG, ret.getReadOnlyGroup()); + } + if (rawRWG != null) { + Util.parseArrayGroupURI(rawRWG, ret.getReadWriteGroup()); + } + if (rawProps != null) { + Util.parseArrayProps(rawProps, ret.getProperties()); + } + + InventoryUtil.assignLastModified(ret, lastModified); + InventoryUtil.assignMetaChecksum(ret, metaChecksum); + + return ret; + } + + private class ObsoleteStorageLocationExtractor implements ResultSetExtractor { final Calendar utc = Calendar.getInstance(DateUtil.UTC); @Override - public Object extractData(ResultSet rs) throws SQLException, DataAccessException { + public ObsoleteStorageLocation extractData(ResultSet rs) throws SQLException, DataAccessException { if (!rs.next()) { return null; } @@ -1332,7 +1809,7 @@ public Object extractData(ResultSet rs) throws SQLException, DataAccessException StorageLocation s = new StorageLocation(storLoc); s.storageBucket = storBucket; - Entity ret = new ObsoleteStorageLocation(id, s); + ObsoleteStorageLocation ret = new ObsoleteStorageLocation(id, s); InventoryUtil.assignLastModified(ret, lastModified); InventoryUtil.assignMetaChecksum(ret, metaChecksum); return ret; @@ -1465,4 +1942,24 @@ public StorageLocationEvent extractData(ResultSet rs) throws SQLException, DataA return ret; } } + + private class NodeExtractor implements ResultSetExtractor { + private ContainerNode parent; + final Calendar utc = Calendar.getInstance(DateUtil.UTC); + + public NodeExtractor(ContainerNode parent) { + this.parent = parent; // optional + } + + @Override + public Node extractData(ResultSet rs) throws SQLException, DataAccessException { + if (!rs.next()) { + return null; + } + + return mapRowToNode(rs, utc, parent); + } + + + } } diff --git a/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/Util.java b/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/Util.java index 404841ff8..782cb8a6f 100644 --- a/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/Util.java +++ b/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/Util.java @@ -77,8 +77,12 @@ import java.sql.SQLException; import java.util.Calendar; import java.util.Date; +import java.util.Set; +import java.util.StringTokenizer; import java.util.UUID; import org.apache.log4j.Logger; +import org.opencadc.gms.GroupURI; +import org.opencadc.vospace.NodeProperty; /** * @@ -378,39 +382,116 @@ public static byte[] getByteArray(ResultSet rs, int col) throw new UnsupportedOperationException("converting " + o.getClass().getName() + " " + o + " to byte[]"); } - /* - * public static int[] getIntArray(ResultSet rs, int col) - * throws SQLException - * { - * Object o = rs.getObject(col); - * return toIntArray(o); - * } - * - * static int[] toIntArray(Object o) - * throws SQLException - * { - * if (o == null) - * return null; - * if (o instanceof Array) - * { - * Array a = (Array) o; - * o = a.getArray(); - * } - * if (o instanceof int[]) - * return (int[]) o; - * if (o instanceof byte[]) - * return CaomUtil.decodeIntArray((byte[]) o); - * if (o instanceof Integer[]) - * { - * Integer[] ia = (Integer[]) o; - * int[] ret = new int[ia.length]; - * for (int i=0; i dest) { + // postgresql 1D array: {a,"b,c"} + if (val == null || val.isEmpty()) { + return; + } + // GroupURI names can contain alphanumeric,comma,dash,dot,underscore,~ + // PG quotes them if comma is present (eg in the group name) + char delim = '"'; + int i = 0; + int j = val.indexOf(delim); + while (j != -1) { + String token = val.substring(i, j); + //log.warn("token: " + i + "," + j + " " + token); + i = j + 1; + j = val.indexOf(delim, i); + + handleToken(token, dest); + } + String token = val.substring(i); + //log.warn("token: " + i + " " + token); + handleToken(token, dest); + } + private static void handleToken(String token, Set dest) { + if (token.startsWith("ivo://")) { + dest.add(new GroupURI(URI.create(token))); + } else { + StringTokenizer st = new StringTokenizer(token, "{,}"); + while (st.hasMoreTokens()) { + String s = st.nextToken(); + dest.add(new GroupURI(URI.create(s))); + } + } + } + + // fills the dest set + public static void parseArrayProps(String val, Set dest) { + // postgresql 2D array: {{a,b},{c,d}} + if (val == null || val.isEmpty()) { + return; + } + char open = '{'; + char close = '}'; + char quote = '"'; + int i = val.indexOf(open); + int j = val.lastIndexOf(close); + if (j > i) { + val = val.substring(i + 1, j); + } + i = val.indexOf(open); + j = val.indexOf(close, i + 1); + int k = 0; + while (i != -1 && j != -1 && k++ < 20) { + String t1 = val.substring(i + 1, j); + //log.warn("\tt1: " + i + "," + j + " " + t1); + handleProp(t1, dest); + + if (i != -1 && j > 0) { + i = val.indexOf(open, j); + j = val.indexOf(close, i + 1); + // look ahead for quotes + int q = val.indexOf(quote, i); + //log.warn("i=" + i + " j=" + j + " q=" + q); + if (q != -1 && q < j) { + int cq = val.indexOf(quote, q + 1); + j = val.indexOf(close, cq); + //log.warn("\tcq=" + cq + " j=" + j); + } + } + } + } + + private static void handleProp(String token, Set dest) { + int q = token.indexOf('"'); + int cq = -1; + if (q == -1) { + q = Integer.MAX_VALUE; + } else { + cq = token.indexOf('"', q + 1); + } + int c = token.indexOf(','); + + String key; + int split = c; + if (c < q) { + // key + key = token.substring(0, c); + } else { + // "key" + key = token.substring(q + 1, cq); + split = cq + 1; + } + //log.warn("\tkey: " + key); + + q = token.indexOf('"', split + 1); + cq = -1; + if (q == -1) { + q = Integer.MAX_VALUE; + } else { + cq = token.indexOf('"', q + 1); + } + String val; + if (token.length() < q) { + val = token.substring(split + 1); + } else { + val = token.substring(q + 1, cq); + } + //log.warn("\tval: " + val); + + dest.add(new NodeProperty(URI.create(key), val)); + } } diff --git a/cadc-inventory-db/src/main/java/org/opencadc/vospace/db/InitDatabaseVOS.java b/cadc-inventory-db/src/main/java/org/opencadc/vospace/db/InitDatabaseVOS.java new file mode 100644 index 000000000..093618755 --- /dev/null +++ b/cadc-inventory-db/src/main/java/org/opencadc/vospace/db/InitDatabaseVOS.java @@ -0,0 +1,113 @@ +/* +************************************************************************ +******************* CANADIAN ASTRONOMY DATA CENTRE ******************* +************** CENTRE CANADIEN DE DONNÉES ASTRONOMIQUES ************** +* +* (c) 2023. (c) 2023. +* Government of Canada Gouvernement du Canada +* National Research Council Conseil national de recherches +* Ottawa, Canada, K1A 0R6 Ottawa, Canada, K1A 0R6 +* All rights reserved Tous droits réservés +* +* NRC disclaims any warranties, Le CNRC dénie toute garantie +* expressed, implied, or énoncée, implicite ou légale, +* statutory, of any kind with de quelque nature que ce +* respect to the software, soit, concernant le logiciel, +* including without limitation y compris sans restriction +* any warranty of merchantability toute garantie de valeur +* or fitness for a particular marchande ou de pertinence +* purpose. NRC shall not be pour un usage particulier. +* liable in any event for any Le CNRC ne pourra en aucun cas +* damages, whether direct or être tenu responsable de tout +* indirect, special or general, dommage, direct ou indirect, +* consequential or incidental, particulier ou général, +* arising from the use of the accessoire ou fortuit, résultant +* software. Neither the name de l'utilisation du logiciel. Ni +* of the National Research le nom du Conseil National de +* Council of Canada nor the Recherches du Canada ni les noms +* names of its contributors may de ses participants ne peuvent +* be used to endorse or promote être utilisés pour approuver ou +* products derived from this promouvoir les produits dérivés +* software without specific prior de ce logiciel sans autorisation +* written permission. préalable et particulière +* par écrit. +* +* This file is part of the Ce fichier fait partie du projet +* OpenCADC project. OpenCADC. +* +* OpenCADC is free software: OpenCADC est un logiciel libre ; +* you can redistribute it and/or vous pouvez le redistribuer ou le +* modify it under the terms of modifier suivant les termes de +* the GNU Affero General Public la “GNU Affero General Public +* License as published by the License” telle que publiée +* Free Software Foundation, par la Free Software Foundation +* either version 3 of the : soit la version 3 de cette +* License, or (at your option) licence, soit (à votre gré) +* any later version. toute version ultérieure. +* +* OpenCADC is distributed in the OpenCADC est distribué +* hope that it will be useful, dans l’espoir qu’il vous +* but WITHOUT ANY WARRANTY; sera utile, mais SANS AUCUNE +* without even the implied GARANTIE : sans même la garantie +* warranty of MERCHANTABILITY implicite de COMMERCIALISABILITÉ +* or FITNESS FOR A PARTICULAR ni d’ADÉQUATION À UN OBJECTIF +* PURPOSE. See the GNU Affero PARTICULIER. Consultez la Licence +* General Public License for Générale Publique GNU Affero +* more details. pour plus de détails. +* +* You should have received Vous devriez avoir reçu une +* a copy of the GNU Affero copie de la Licence Générale +* General Public License along Publique GNU Affero avec +* with OpenCADC. If not, see OpenCADC ; si ce n’est +* . pas le cas, consultez : +* . +* +************************************************************************ +*/ + +package org.opencadc.vospace.db; + +import java.net.URL; +import javax.sql.DataSource; +import org.apache.log4j.Logger; +import org.opencadc.inventory.db.version.InitDatabase; + +/** + * + * @author pdowler + */ +public class InitDatabaseVOS extends ca.nrc.cadc.db.version.InitDatabase { + private static final Logger log = Logger.getLogger(InitDatabaseVOS.class); + + public static final String MODEL_NAME = "vospace-inventory"; + public static final String MODEL_VERSION = "0.1"; + public static final String PREV_MODEL_VERSION = "n/a"; + //public static final String PREV_MODEL_VERSION = "DO-NOT_UPGRADE-BY-ACCIDENT"; + + static String[] CREATE_SQL = new String[] { + "vos.ModelVersion.sql", + "vos.Node.sql", + "vos.DeletedNodeEvent.sql", + "inventory.permissions.sql" + }; + + static String[] UPGRADE_SQL = new String[] { + "inventory.permissions.sql" + }; + + public InitDatabaseVOS(DataSource ds, String database, String schema) { + super(ds, database, schema, MODEL_NAME, MODEL_VERSION, PREV_MODEL_VERSION); + for (String s : CREATE_SQL) { + createSQL.add(s); + } + for (String s : UPGRADE_SQL) { + upgradeSQL.add(s); + } + } + + @Override + protected URL findSQL(String fname) { + // SQL files are stored inside the jar file + return InitDatabase.class.getClassLoader().getResource(fname); + } +} diff --git a/cadc-inventory-db/src/main/java/org/opencadc/vospace/db/NodeDAO.java b/cadc-inventory-db/src/main/java/org/opencadc/vospace/db/NodeDAO.java new file mode 100644 index 000000000..acf81b961 --- /dev/null +++ b/cadc-inventory-db/src/main/java/org/opencadc/vospace/db/NodeDAO.java @@ -0,0 +1,180 @@ +/* +************************************************************************ +******************* CANADIAN ASTRONOMY DATA CENTRE ******************* +************** CENTRE CANADIEN DE DONNÉES ASTRONOMIQUES ************** +* +* (c) 2023. (c) 2023. +* Government of Canada Gouvernement du Canada +* National Research Council Conseil national de recherches +* Ottawa, Canada, K1A 0R6 Ottawa, Canada, K1A 0R6 +* All rights reserved Tous droits réservés +* +* NRC disclaims any warranties, Le CNRC dénie toute garantie +* expressed, implied, or énoncée, implicite ou légale, +* statutory, of any kind with de quelque nature que ce +* respect to the software, soit, concernant le logiciel, +* including without limitation y compris sans restriction +* any warranty of merchantability toute garantie de valeur +* or fitness for a particular marchande ou de pertinence +* purpose. NRC shall not be pour un usage particulier. +* liable in any event for any Le CNRC ne pourra en aucun cas +* damages, whether direct or être tenu responsable de tout +* indirect, special or general, dommage, direct ou indirect, +* consequential or incidental, particulier ou général, +* arising from the use of the accessoire ou fortuit, résultant +* software. Neither the name de l'utilisation du logiciel. Ni +* of the National Research le nom du Conseil National de +* Council of Canada nor the Recherches du Canada ni les noms +* names of its contributors may de ses participants ne peuvent +* be used to endorse or promote être utilisés pour approuver ou +* products derived from this promouvoir les produits dérivés +* software without specific prior de ce logiciel sans autorisation +* written permission. préalable et particulière +* par écrit. +* +* This file is part of the Ce fichier fait partie du projet +* OpenCADC project. OpenCADC. +* +* OpenCADC is free software: OpenCADC est un logiciel libre ; +* you can redistribute it and/or vous pouvez le redistribuer ou le +* modify it under the terms of modifier suivant les termes de +* the GNU Affero General Public la “GNU Affero General Public +* License as published by the License” telle que publiée +* Free Software Foundation, par la Free Software Foundation +* either version 3 of the : soit la version 3 de cette +* License, or (at your option) licence, soit (à votre gré) +* any later version. toute version ultérieure. +* +* OpenCADC is distributed in the OpenCADC est distribué +* hope that it will be useful, dans l’espoir qu’il vous +* but WITHOUT ANY WARRANTY; sera utile, mais SANS AUCUNE +* without even the implied GARANTIE : sans même la garantie +* warranty of MERCHANTABILITY implicite de COMMERCIALISABILITÉ +* or FITNESS FOR A PARTICULAR ni d’ADÉQUATION À UN OBJECTIF +* PURPOSE. See the GNU Affero PARTICULIER. Consultez la Licence +* General Public License for Générale Publique GNU Affero +* more details. pour plus de détails. +* +* You should have received Vous devriez avoir reçu une +* a copy of the GNU Affero copie de la Licence Générale +* General Public License along Publique GNU Affero avec +* with OpenCADC. If not, see OpenCADC ; si ce n’est +* . pas le cas, consultez : +* . +* +************************************************************************ +*/ + +package org.opencadc.vospace.db; + +import ca.nrc.cadc.io.ResourceIterator; +import java.util.UUID; +import org.apache.log4j.Logger; +import org.opencadc.inventory.db.AbstractDAO; +import org.opencadc.inventory.db.SQLGenerator; +import org.opencadc.vospace.ContainerNode; +import org.opencadc.vospace.Node; +import org.springframework.jdbc.BadSqlGrammarException; +import org.springframework.jdbc.core.JdbcTemplate; + +/** + * + * @author pdowler + */ +public class NodeDAO extends AbstractDAO { + private static final Logger log = Logger.getLogger(NodeDAO.class); + + public NodeDAO() { + super(true); + } + + @Override + public void put(Node val) { + // TBD: caller can assign parent or parentID before put, but here we need + // parentID and it must be assigned before metaChecksum compute in super.put() + if (val.parentID == null && val.parent != null) { + val.parentID = val.parent.getID(); + } + super.put(val); + } + + @Override + public Node lock(Node n) { + if (n == null) { + throw new IllegalArgumentException("entity cannot be null"); + } + // override because Node has subclasses: force base class here + return super.lock(Node.class, n.getID()); + } + + public Node get(UUID id) { + return super.get(Node.class, id); + } + + public Node get(ContainerNode parent, String name) { + checkInit(); + log.debug("GET: " + parent.getID() + " + " + name); + long t = System.currentTimeMillis(); + + try { + JdbcTemplate jdbc = new JdbcTemplate(dataSource); + SQLGenerator.NodeGet get = (SQLGenerator.NodeGet) gen.getEntityGet(Node.class); + get.setPath(parent, name); + return get.execute(jdbc); + } catch (BadSqlGrammarException ex) { + handleInternalFail(ex); + } finally { + long dt = System.currentTimeMillis() - t; + log.debug("GET: " + parent.getID() + " + " + name + " " + dt + "ms"); + } + throw new RuntimeException("BUG: handleInternalFail did not throw"); + } + + public boolean isEmpty(ContainerNode parent) { + checkInit(); + log.debug("isEmpty: " + parent.getID()); + long t = System.currentTimeMillis(); + + try { + JdbcTemplate jdbc = new JdbcTemplate(dataSource); + SQLGenerator.NodeCount count = (SQLGenerator.NodeCount) gen.getNodeCount(); + count.setID(parent.getID()); + int num = count.execute(jdbc); + return (num == 0); + } catch (BadSqlGrammarException ex) { + handleInternalFail(ex); + } finally { + long dt = System.currentTimeMillis() - t; + log.debug("isEmpty: " + parent.getID() + " " + dt + "ms"); + } + throw new RuntimeException("BUG: handleInternalFail did not throw"); + } + + public void delete(UUID id) { + super.delete(Node.class, id); + } + + public ResourceIterator iterator(ContainerNode parent, Integer limit, String start) { + if (parent == null) { + throw new IllegalArgumentException("childIterator: parent cannot be null"); + } + log.debug("iterator: " + parent.getID()); + + checkInit(); + long t = System.currentTimeMillis(); + + try { + SQLGenerator.NodeIteratorQuery iter = (SQLGenerator.NodeIteratorQuery) gen.getEntityIteratorQuery(Node.class); + iter.setParent(parent); + iter.setStart(start); + iter.setLimit(limit); + return iter.query(dataSource); + } catch (BadSqlGrammarException ex) { + handleInternalFail(ex); + } finally { + long dt = System.currentTimeMillis() - t; + log.debug("iterator: " + parent.getID() + " " + dt + "ms"); + } + throw new RuntimeException("BUG: should be unreachable"); + } +} diff --git a/cadc-inventory-db/src/main/resources/vos.DeletedNodeEvent.sql b/cadc-inventory-db/src/main/resources/vos.DeletedNodeEvent.sql new file mode 100644 index 000000000..341a670e5 --- /dev/null +++ b/cadc-inventory-db/src/main/resources/vos.DeletedNodeEvent.sql @@ -0,0 +1,17 @@ + +create table .DeletedNodeEvent ( + -- type is immutable + nodeType char(1) not null, + + -- support cleanup of obsolete artifacts + storageID varchar(512), + + lastModified timestamp not null, + metaChecksum varchar(136) not null, + id uuid not null primary key +); + + + +create index dne_lastmodified on .DeletedNodeEvent(lastModified); + diff --git a/cadc-inventory-db/src/main/resources/vos.ModelVersion.sql b/cadc-inventory-db/src/main/resources/vos.ModelVersion.sql new file mode 100644 index 000000000..ca9126697 --- /dev/null +++ b/cadc-inventory-db/src/main/resources/vos.ModelVersion.sql @@ -0,0 +1,9 @@ + +create table .ModelVersion +( + model varchar(32) not null primary key, + version varchar(32) not null, + lastModified timestamp not null +) +; + diff --git a/cadc-inventory-db/src/main/resources/vos.Node.sql b/cadc-inventory-db/src/main/resources/vos.Node.sql new file mode 100644 index 000000000..e52a5ed8c --- /dev/null +++ b/cadc-inventory-db/src/main/resources/vos.Node.sql @@ -0,0 +1,34 @@ + +create table .Node ( + -- require a special root ID value but prevent bugs + parentID uuid not null, + name varchar(512) not null, + nodeType char(1) not null, + + ownerID varchar(256) not null, + isPublic boolean, + isLocked boolean, + readOnlyGroups text, + readWriteGroups text, + + -- store all props in a 2D array + properties text[][], + + -- ContainerNode + inheritPermissions boolean, + + -- DataNode + busy boolean, + storageID varchar(512), + + -- LinkNode + target text, + + lastModified timestamp not null, + metaChecksum varchar(136) not null, + id uuid not null primary key +); + +create unique index node_parent_child on .Node(parentID,name); + +create index node_lastmodified on .Node(lastModified); diff --git a/cadc-inventory-db/src/test/java/org/opencadc/inventory/db/UtilTest.java b/cadc-inventory-db/src/test/java/org/opencadc/inventory/db/UtilTest.java new file mode 100644 index 000000000..c26cf3f26 --- /dev/null +++ b/cadc-inventory-db/src/test/java/org/opencadc/inventory/db/UtilTest.java @@ -0,0 +1,159 @@ +/* +************************************************************************ +******************* CANADIAN ASTRONOMY DATA CENTRE ******************* +************** CENTRE CANADIEN DE DONNÉES ASTRONOMIQUES ************** +* +* (c) 2023. (c) 2023. +* Government of Canada Gouvernement du Canada +* National Research Council Conseil national de recherches +* Ottawa, Canada, K1A 0R6 Ottawa, Canada, K1A 0R6 +* All rights reserved Tous droits réservés +* +* NRC disclaims any warranties, Le CNRC dénie toute garantie +* expressed, implied, or énoncée, implicite ou légale, +* statutory, of any kind with de quelque nature que ce +* respect to the software, soit, concernant le logiciel, +* including without limitation y compris sans restriction +* any warranty of merchantability toute garantie de valeur +* or fitness for a particular marchande ou de pertinence +* purpose. NRC shall not be pour un usage particulier. +* liable in any event for any Le CNRC ne pourra en aucun cas +* damages, whether direct or être tenu responsable de tout +* indirect, special or general, dommage, direct ou indirect, +* consequential or incidental, particulier ou général, +* arising from the use of the accessoire ou fortuit, résultant +* software. Neither the name de l'utilisation du logiciel. Ni +* of the National Research le nom du Conseil National de +* Council of Canada nor the Recherches du Canada ni les noms +* names of its contributors may de ses participants ne peuvent +* be used to endorse or promote être utilisés pour approuver ou +* products derived from this promouvoir les produits dérivés +* software without specific prior de ce logiciel sans autorisation +* written permission. préalable et particulière +* par écrit. +* +* This file is part of the Ce fichier fait partie du projet +* OpenCADC project. OpenCADC. +* +* OpenCADC is free software: OpenCADC est un logiciel libre ; +* you can redistribute it and/or vous pouvez le redistribuer ou le +* modify it under the terms of modifier suivant les termes de +* the GNU Affero General Public la “GNU Affero General Public +* License as published by the License” telle que publiée +* Free Software Foundation, par la Free Software Foundation +* either version 3 of the : soit la version 3 de cette +* License, or (at your option) licence, soit (à votre gré) +* any later version. toute version ultérieure. +* +* OpenCADC is distributed in the OpenCADC est distribué +* hope that it will be useful, dans l’espoir qu’il vous +* but WITHOUT ANY WARRANTY; sera utile, mais SANS AUCUNE +* without even the implied GARANTIE : sans même la garantie +* warranty of MERCHANTABILITY implicite de COMMERCIALISABILITÉ +* or FITNESS FOR A PARTICULAR ni d’ADÉQUATION À UN OBJECTIF +* PURPOSE. See the GNU Affero PARTICULIER. Consultez la Licence +* General Public License for Générale Publique GNU Affero +* more details. pour plus de détails. +* +* You should have received Vous devriez avoir reçu une +* a copy of the GNU Affero copie de la Licence Générale +* General Public License along Publique GNU Affero avec +* with OpenCADC. If not, see OpenCADC ; si ce n’est +* . pas le cas, consultez : +* . +* +************************************************************************ +*/ + +package org.opencadc.inventory.db; + +import ca.nrc.cadc.util.Log4jInit; +import java.util.Set; +import java.util.TreeSet; +import org.apache.log4j.Level; +import org.apache.log4j.Logger; +import org.junit.Test; +import org.opencadc.gms.GroupURI; +import org.opencadc.vospace.NodeProperty; + +/** + * + * @author pdowler + */ +public class UtilTest { + private static final Logger log = Logger.getLogger(UtilTest.class); + + static { + Log4jInit.setLevel("org.opencadc.inventory.db", Level.INFO); + } + + public UtilTest() { + } + + @Test + public void testParseArrayGroupURI() throws Exception { + + String str = "{ivo://opencadc.org/gms?g3," + + "ivo://opencadc.org/gms?g6-g7," + + "ivo://opencadc.org/gms?g6.g7," + + "ivo://opencadc.org/gms?g6_g7," + + "ivo://opencadc.org/gms?g6~g7}"; + + Set dest = new TreeSet<>(); + Util.parseArrayGroupURI(str, dest); + for (GroupURI u : dest) { + log.info("uri: " + u.getURI()); + } + } + + @Test + public void testParseArrayNodeProperty() throws Exception { + + String str = ""; + Set dest = new TreeSet<>(); + + log.info("raw:\n" + str + "\n"); + Util.parseArrayProps(str, dest); + for (NodeProperty p : dest) { + log.info("prop: " + p.getKey() + " = " + p.getValue()); + } + + str = "{{ivo://ivoa.net/vospace/core#description,stuff}}"; + dest.clear(); + log.info("raw:\n" + str + "\n"); + Util.parseArrayProps(str, dest); + for (NodeProperty p : dest) { + log.info("prop: " + p.getKey() + " = " + p.getValue()); + } + + str = "{{ivo://ivoa.net/vospace/core#description,\"this is the good stuff(tm)\"}}"; + dest.clear(); + log.info("raw:\n" + str + "\n"); + Util.parseArrayProps(str, dest); + for (NodeProperty p : dest) { + log.info("prop: " + p.getKey() + " = " + p.getValue()); + } + + str = "{{ivo://ivoa.net/vospace/core#description,\"this is the good stuff(tm)\"}," + + "{ivo://ivoa.net/vospace/core#type,text/plain}}"; + dest.clear(); + log.info("raw:\n" + str + "\n"); + Util.parseArrayProps(str, dest); + for (NodeProperty p : dest) { + log.info("prop: " + p.getKey() + " = " + p.getValue()); + } + + str = "{{custom:prop,\"spaces in value\"}," + + "{ivo://ivoa.net/vospace/core#length,123}," + + "{ivo://ivoa.net/vospace/core#type,text/plain}," + + "{\"sketchy:a,b\",comma-in-uri}," + + "{sketchy:funny,\"value,with,{delims}\"}}"; + + dest.clear(); + log.info("raw:\n" + str + "\n"); + Util.parseArrayProps(str, dest); + for (NodeProperty p : dest) { + log.info("prop: " + p.getKey() + " = " + p.getValue()); + } + } +} diff --git a/vault-quota/Design.md b/vault-quota/Design.md new file mode 100644 index 000000000..b2b43b920 --- /dev/null +++ b/vault-quota/Design.md @@ -0,0 +1,109 @@ +# vault quota design/algorithms + +The definitive source of content-length (file size) of a DataNode comes from the +`inventory.Artifact` table and it not known until a PUT to storage is completed. +In the case of a `vault` service co-located with a single storage site (`minoc`), +the new Artifact is visible in the database as soon as the PUT to `minoc` is +completed. In the case of a `vault` service co-located with a global SI, the new +Artifact is visible in the database once it is synced from the site of the PUT to +`minoc` to the global database by `fenwick` (or worst case: `ratik`). + +## TODO +The design below only takes into account incremental propagation of space used +by stored files. It is not complete/verified until we also come up with a validation +algorithm that can detect and fix discrepancies in a live `vault`. + +## DataNode size algorithm: +This is an event watcher that gets Artifact events (after a PUT) and intiates the +propagation of sizes (space used). +``` +track progress using HarvestState (source: `db:{bucket range}`, name: TBD) +incremental query for new artifacts in lastModified order +for each new Artifact: + query for DataNode (storageID = artifact.uri) + if Artifact.contentLength != Node.size: + start txn + lock datanode + compute delta + lock parent + apply delta to parent.delta + set dataNode.size + update HarvestState + commit txn +``` +Optimization: The above sequence does the first step of propagation from DataNode to +parent ContainerNode so the maximum work can be done in parallel using bucket ranges +(smaller than 0-f). It also means the propagation below only has to consider +ContainerNode.delta since DataNode(s) never have a delta. + +## ContainerNode size propagation algorithm: +``` +query for ContainerNode with non-zero delta +for each ContainerNode: + start txn + lock containernode + re-check delta + lock parent + apply delta to parent.delta + apply delta containernode.size, set containernode.delta=0 + commit txn +``` +The above sequence finds candidate propagations, locks (order: child-then-parent as above), +and applies the propagation. This moves the outstanding delta up the tree one level. If the +sequence acts on multiple child containers before the parent, the delta(s) naturally +_merge_ and there are fewer larger delta propagations in the upper part of the tree. It would +be optimal to do propagations depth-first but it doesn't seem practical to forcibly accomplish +that ordering. + +Container size propagation will be implemented as a single sequence (thread). We could add +something to the vospace.Node table to support subdividing work and enable multiple threads, +but there is nothing there right now. + +## validation + +### DataNode vs Artifact discrepancies +These can be validated in parallel by multiple threads, subdivide work by bucket. + +``` +discrepancy 1: Artifact exists but DataNode does not +explanation: DataNode created, transfer negotiated, DataNode removed, transfer executed +evidence: check for DeletedNodeEvent +action: remove artifact, create DeletedArtifactEvent +else: ?? + +discrepancy 2: DataNode exists but Artifact does not +explanation: DataNode created, Artifact never (successfully) put +evidence: dataNode.size == 0 +action: none + +discrepancy 3: DataNode exists but Artifact does not +explanation: deleted or lost Artifact +evidence: DataNode.size != 0 (deleted vs lost: DeletedArtifactEvent exists) +action: fix DataNode.size + +discrepancy 4: DataNode.size != Artifact.contentLength +explanation: pending/missed Artifact event +action: fix DataNode and propagate delta to parent ContainerNode (same as incremental) +``` + +This could be accomplished with a single query on on inventory.Artifact full outer join +vospace.Node to get all the pairs. The more generic approach would be to do a merge join +of two iterators: + +Iterator aiter = artifactDAO.iterator(vaultNamespace, bucket); +Iterator niter = nodeDAO.iterator(vaultNamespace, bucket); + +The more generic dual iterator approach could be made to work if the inventory and vospace +content are in different PG database or server - TBD. + +## database changes required +note: all field and column names TBD +* add `size` and `delta` fields to ContainerNode (transient) +* add `size` field to DataNode (transient) +* add `size` to the `vospace.Node` table +* add `delta` to the `vospace.Node` table +* add `storageBucket` to `vospace.Node` table (validation) +* incremental sync query/iterator (ArtifactDAO?) +* lookup DataNode by storageID (ArtifactDAO?) +* indices to support new queries + diff --git a/vault-quota/README.md b/vault-quota/README.md new file mode 100644 index 000000000..c0f2db0eb --- /dev/null +++ b/vault-quota/README.md @@ -0,0 +1,59 @@ +# Storage Inventory VOSpace quota support process (vault-quota) + +Process to maintain container node sizes so that quota limits can be enforced by the +main `vault` service. This process runs in incremental mode (single process running +continuously) to update a local vospace database. + +`vault-quota` is an optional process that is only needed if `vault` is configured to +enforce quotas, although it could be used to maintain container node sizes without +quota enforcement. + +## configuration +See the [cadc-java](https://github.com/opencadc/docker-base/tree/master/cadc-java) image +docs for general config requirements. + +Runtime configuration must be made available via the `/config` directory. + +### vault-quota.properties +``` +org.opencadc.vault.quota.logging = {info|debug} + +# inventory database settings +org.opencadc.inventory.db.SQLGenerator=org.opencadc.inventory.db.SQLGenerator +org.opencadc.vault.quota.nodes.schema={schema for inventory database objects} +org.opencadc.vault.quota.nodes.username={username for inventory admin} +org.opencadc.vault.quota.nodes.password={password for inventory admin} +org.opencadc.vault.quota.nodes.url=jdbc:postgresql://{server}/{database} + +org.opencadc.vault.quota.threads={number of threads to watch for artifact events} + +# storage namespace +org.opencadc.vault.storage.namespace = {a storage inventory namespace to use} +``` +The _nodes_ account owns and manages (create, alter, drop) vospace database objects and updates +content in the vospace schema. The database is specified in the JDBC URL. Failure to connect or +initialize the database will show up in logs. + +The _threads_ key configures the number of threads that watch for new Artifact events and initiate +the propagation of sizes to parent containers. These threads each monitor a subset of artifacts using +`Artifact.uriBucket` filtering; for simplicity, the following values are allowed: 1, 2, 4, 8, 16. + +In addition to the above threads, there is one additional thread that propagates size changes up +the tree of container nodes to the container node(s) where quotas are specified. + +## building it +``` +gradle clean build +docker build -t vault-quota -f Dockerfile . +``` + +## checking it +``` +docker run -it vault-quota:latest /bin/bash +``` + +## running it +``` +docker run --user opencadc:opencadc -v /path/to/external/config:/config:ro --name vault-quota vault-quota:latest +``` + diff --git a/vault/Dockerfile b/vault/Dockerfile index f3bc94674..4f35f9ff5 100644 --- a/vault/Dockerfile +++ b/vault/Dockerfile @@ -1,3 +1,4 @@ -FROM cadc-tomcat:1 +FROM images.opencadc.org/library/cadc-tomcat:1 + +COPY build/libs/vault.war /usr/share/tomcat/webapps/vault.war -COPY build/libs/vault.war /usr/share/tomcat/webapps/ diff --git a/vault/README.md b/vault/README.md index 3dd77529a..c4615fc99 100644 --- a/vault/README.md +++ b/vault/README.md @@ -1,16 +1,38 @@ -# Storage Inventory storage management service (vault) +# Storage Inventory VOSpace-2.1 service (vault) -## configuration -See the [cadc-tomcat](https://github.com/opencadc/docker-base/tree/master/cadc-tomcat) image docs -for expected deployment and general config requirements. The `vault` war file can be renamed -at deployment time in order to support an alternate service name, including introducing -additional path elements (see war-rename.conf). +The `vault` servcie is an implementation of the IVOA VOSpace +specification designed to co-exist with other storage-inventory components. It provides a heirarchical data +organization laye on top of the storage management of storage-inventory. + +The simplest configuration would be to deploy `vault` with `minoc` with a single metadata database and single +back end storage system. Details: TBD. + +The other option would be to deploy `vault` with `raven` and `luskan` in a global inventory database and make +use of one or more of the network of known storage sites to store files. Details: TBD. + +## deployment + +The `vault` war file can be renamed at deployment time in order to support an alternate service name, +including introducing additional path elements. See +cadc-tomcat (war-rename.conf). -Runtime configuration must be made available via the `/config` directory. +## configuration +The following runtime configuration must be made available via the `/config` directory. ### catalina.properties -When running vault.war in tomcat, parameters of the connection pool in META-INF/context.xml need -to be configured in catalina.properties: +This file contains java system properties to configure the tomcat server and some of the java libraries used in the service. + +See cadc-tomcat +for system properties related to the deployment environment. + +See cadc-util +for common system properties. + +`vault` includes multiple IdentityManager implementations to support authenticated access: +- See cadc-access-control-identity for CADC access-control system support. +- See cadc-gms for OIDC token support. + +`vault` requires a connection pool to the local database: ``` # database connection pools org.opencadc.vault.nodes.maxActive={max connections for vospace pool} @@ -18,34 +40,60 @@ org.opencadc.vault.nodes.username={username for vospace pool} org.opencadc.vault.nodes.password={password for vospace pool} org.opencadc.vault.nodes.url=jdbc:postgresql://{server}/{database} ``` -The `nodes` account owns and manages (create, alter, drop) vault database objects and manages +The _nodes_ account owns and manages (create, alter, drop) vospace database objects and manages all the content (insert, update, delete). The database is specified in the JDBC URL and the schema name is specified in the vault.properties (below). Failure to connect or initialize the database will show up in logs and in the VOSI-availability output. +### cadc-registry.properties + +See cadc-registry. + ### vault.properties A vault.properties file in /config is required to run this service. The following keys are required: ``` # service identity -org.opencadc.vault.resourceID=ivo://{authority}/{name} +org.opencadc.vault.resourceID = ivo://{authority}/{name} # vault database settings -org.opencadc.vault.nodes.schema={schema name} +org.opencadc.vault.inventory.schema = {inventory schema name} +org.opencadc.vault.vospace.schema = {vospace schema name} + +# root container nodes +org.opencadc.vault.root.owner = {owner of root node} + +# storage namespace +org.opencadc.vault.storage.namespace = {a storage inventory namespace to use} ``` The vault _resourceID_ is the resourceID of _this_ vault service. -The nodes _schema_ name is the name of the database schema used for all created database objects (tables, indices, etc). +The _inventory.schema_ name is the name of the database schema that contains the inventory database objects. The account nominally requires read-only (select) permission on those objects. This currently must be "inventory" due to configuration +limitations in luskan. + +The _vospace.schema_ name is the name of the database schema used for all created database objects (tables, indices, etc). Note that with a single connection pool, the two schemas must be in the same database and some operations may join tables +in the two schemas (probably just vospace.node join inventory.artifact). + +The root node owner has full read and write permission in the root container, so it can create and delete container +nodes at the root and assign container node properties that are normally read-only to normal users: owner, quota, +etc. This is probably an X509 distingushed name of the user (to start). **not fully implemented** TBD. + +The _namespace_ configures `vault` to use the specified namespace in storage-inventory to store files. This only +applies to new data nodes that are created and will not effect previously created nodes and artifacts. Probably don't +want to change this... prevent change? TBD. ### vault-availability.properties (optional) -``` -The vault-availability.properties file specifies which users have the authority to change the availability state of the vault service. Each entry consists of a key=value pair. The key is always "users". The value is the x500 canonical user name. -``` + +The vault-availability.properties file specifies which users have the authority to change the availability state of +the vault service. Each entry consists of a key=value pair. The key is always "users". The value is the x500 canonical +user name. Example: ``` users = {user identity} ``` -`users` specifies the user(s) who are authorized to make calls to the service. The value is a list of user identities (X500 distingushed name), one line per user. Optional: if the `vault-availability.properties` is not found or does not list any `users`, the service will function in the default mode (ReadWrite) and the state will not be changeable. +`users` specifies the user(s) who are authorized to make calls to the service. The value is a list of user identities +(X500 distingushed name), one line per user. Optional: if the `vault-availability.properties` is not found or does not +list any `users`, the service will function in the default mode (ReadWrite) and the state will not be changeable. ## building it ``` diff --git a/vault/TODO b/vault/TODO new file mode 100644 index 000000000..1481088d1 --- /dev/null +++ b/vault/TODO @@ -0,0 +1,22 @@ + +* NodePersistenceImpl: review for necessary transactions and locks + +* NodePersistenceImpl: reconcile with NodePersistence API and assign responsibilities +- property update checking +- permission checking +- link node resolution + +* files endpoint: +- if coexist with minoc: generate pre-auth URL to it and redirect +- if coexist with raven: need raven ProtocolsGenerator + +* transfer negotiation: +- review cadc-vos-server and cavern implementations +- probably a complete TransferRunner; maybe separate sync and async runners +- if co-exist with minoc: generate pre-auth URL to it +- if co-exist with raven: need raven ProtocolsGenerator +- figure out if/how vault can have it's own uws tables in db or share with inventory (luskan) + +* pre-auth URL keys -- what to support? recommend? +- vault has it's own key pair && minoc(s) have multiple pub keys? +- vault and raven share private key? diff --git a/vault/build.gradle b/vault/build.gradle index dc09ec290..7fd069259 100644 --- a/vault/build.gradle +++ b/vault/build.gradle @@ -24,16 +24,37 @@ war { include 'VERSION' } } +description = 'OpenCADC VOSpace server' +def git_url = 'https://github.com/opencadc/vos' dependencies { + compile 'javax.servlet:javax.servlet-api:[3.1,4.0)' + + compile 'org.opencadc:cadc-util:[1.9.10,2.0)' compile 'org.opencadc:cadc-log:[1.1.6,2.0)' - compile 'org.opencadc:cadc-vosi:[1.4.3,2.0)' + compile 'org.opencadc:cadc-gms:[1.0.5,)' + compile 'org.opencadc:cadc-rest:[1.3.16,)' + compile 'org.opencadc:cadc-vos:[2.0,)' + compile 'org.opencadc:cadc-vos-server-alt:[2.0,)' + compile 'org.opencadc:cadc-vosi:[1.3.2,)' + compile 'org.opencadc:cadc-uws:[1.0,)' + compile 'org.opencadc:cadc-uws-server:[1.2.19,)' + compile 'org.opencadc:cadc-access-control:[1.1.1,2.0)' + compile 'org.opencadc:cadc-cdp:[1.2.3,)' + compile 'org.opencadc:cadc-registry:[1.7.4,)' + compile 'org.opencadc:cadc-inventory:[0.9.4,1.0)' + compile 'org.opencadc:cadc-inventory-db:[0.15.0,1.0)' testCompile 'junit:junit:[4.0,)' + runtime 'org.opencadc:cadc-access-control-identity:[1.2.1,)' + runtime 'org.opencadc:cadc-gms:[1.0.5,)' + intTestCompile 'org.opencadc:cadc-test-vosi:[1.0.11,)' + intTestCompile 'org.opencadc:cadc-test-vos:[2.0,3.0)' } configurations { + compile.exclude group: 'org.restlet.jee' runtime.exclude group: 'org.postgresql:postgresql' } diff --git a/vault/src/intTest/java/org/opencadc/vault/NodesTest.java b/vault/src/intTest/java/org/opencadc/vault/NodesTest.java new file mode 100644 index 000000000..2a4a317da --- /dev/null +++ b/vault/src/intTest/java/org/opencadc/vault/NodesTest.java @@ -0,0 +1,93 @@ +/* +************************************************************************ +******************* CANADIAN ASTRONOMY DATA CENTRE ******************* +************** CENTRE CANADIEN DE DONNÉES ASTRONOMIQUES ************** +* +* (c) 2023. (c) 2023. +* Government of Canada Gouvernement du Canada +* National Research Council Conseil national de recherches +* Ottawa, Canada, K1A 0R6 Ottawa, Canada, K1A 0R6 +* All rights reserved Tous droits réservés +* +* NRC disclaims any warranties, Le CNRC dénie toute garantie +* expressed, implied, or énoncée, implicite ou légale, +* statutory, of any kind with de quelque nature que ce +* respect to the software, soit, concernant le logiciel, +* including without limitation y compris sans restriction +* any warranty of merchantability toute garantie de valeur +* or fitness for a particular marchande ou de pertinence +* purpose. NRC shall not be pour un usage particulier. +* liable in any event for any Le CNRC ne pourra en aucun cas +* damages, whether direct or être tenu responsable de tout +* indirect, special or general, dommage, direct ou indirect, +* consequential or incidental, particulier ou général, +* arising from the use of the accessoire ou fortuit, résultant +* software. Neither the name de l'utilisation du logiciel. Ni +* of the National Research le nom du Conseil National de +* Council of Canada nor the Recherches du Canada ni les noms +* names of its contributors may de ses participants ne peuvent +* be used to endorse or promote être utilisés pour approuver ou +* products derived from this promouvoir les produits dérivés +* software without specific prior de ce logiciel sans autorisation +* written permission. préalable et particulière +* par écrit. +* +* This file is part of the Ce fichier fait partie du projet +* OpenCADC project. OpenCADC. +* +* OpenCADC is free software: OpenCADC est un logiciel libre ; +* you can redistribute it and/or vous pouvez le redistribuer ou le +* modify it under the terms of modifier suivant les termes de +* the GNU Affero General Public la “GNU Affero General Public +* License as published by the License” telle que publiée +* Free Software Foundation, par la Free Software Foundation +* either version 3 of the : soit la version 3 de cette +* License, or (at your option) licence, soit (à votre gré) +* any later version. toute version ultérieure. +* +* OpenCADC is distributed in the OpenCADC est distribué +* hope that it will be useful, dans l’espoir qu’il vous +* but WITHOUT ANY WARRANTY; sera utile, mais SANS AUCUNE +* without even the implied GARANTIE : sans même la garantie +* warranty of MERCHANTABILITY implicite de COMMERCIALISABILITÉ +* or FITNESS FOR A PARTICULAR ni d’ADÉQUATION À UN OBJECTIF +* PURPOSE. See the GNU Affero PARTICULIER. Consultez la Licence +* General Public License for Générale Publique GNU Affero +* more details. pour plus de détails. +* +* You should have received Vous devriez avoir reçu une +* a copy of the GNU Affero copie de la Licence Générale +* General Public License along Publique GNU Affero avec +* with OpenCADC. If not, see OpenCADC ; si ce n’est +* . pas le cas, consultez : +* . +* +************************************************************************ +*/ + +package org.opencadc.vault; + +import ca.nrc.cadc.util.Log4jInit; +import java.net.URI; +import org.apache.log4j.Level; +import org.apache.log4j.Logger; +import org.opencadc.gms.GroupURI; + +/** + * Test the nodes endpoint. + * + * @author pdowler + */ +public class NodesTest extends org.opencadc.conformance.vos.NodesTest { + private static final Logger log = Logger.getLogger(NodesTest.class); + + static { + Log4jInit.setLevel("org.opencadc.conformance.vos", Level.DEBUG); + Log4jInit.setLevel("org.opencadc.vospace", Level.DEBUG); + } + + public NodesTest() { + //super(URI.create("ivo://opencadc.org/vault"), "vault-test.pem", new GroupURI(URI.create("ivo://cadc.nrc.ca/gms?CADC_TEST_GROUP2")), "vault-test-auth.pem"); + super(URI.create("ivo://opencadc.org/vault"), "vault-test.pem"); + } +} diff --git a/vault/src/main/java/org/opencadc/vault/NodePersistenceImpl.java b/vault/src/main/java/org/opencadc/vault/NodePersistenceImpl.java new file mode 100644 index 000000000..63bf5ec96 --- /dev/null +++ b/vault/src/main/java/org/opencadc/vault/NodePersistenceImpl.java @@ -0,0 +1,546 @@ +/* +************************************************************************ +******************* CANADIAN ASTRONOMY DATA CENTRE ******************* +************** CENTRE CANADIEN DE DONNÉES ASTRONOMIQUES ************** +* +* (c) 2023. (c) 2023. +* Government of Canada Gouvernement du Canada +* National Research Council Conseil national de recherches +* Ottawa, Canada, K1A 0R6 Ottawa, Canada, K1A 0R6 +* All rights reserved Tous droits réservés +* +* NRC disclaims any warranties, Le CNRC dénie toute garantie +* expressed, implied, or énoncée, implicite ou légale, +* statutory, of any kind with de quelque nature que ce +* respect to the software, soit, concernant le logiciel, +* including without limitation y compris sans restriction +* any warranty of merchantability toute garantie de valeur +* or fitness for a particular marchande ou de pertinence +* purpose. NRC shall not be pour un usage particulier. +* liable in any event for any Le CNRC ne pourra en aucun cas +* damages, whether direct or être tenu responsable de tout +* indirect, special or general, dommage, direct ou indirect, +* consequential or incidental, particulier ou général, +* arising from the use of the accessoire ou fortuit, résultant +* software. Neither the name de l'utilisation du logiciel. Ni +* of the National Research le nom du Conseil National de +* Council of Canada nor the Recherches du Canada ni les noms +* names of its contributors may de ses participants ne peuvent +* be used to endorse or promote être utilisés pour approuver ou +* products derived from this promouvoir les produits dérivés +* software without specific prior de ce logiciel sans autorisation +* written permission. préalable et particulière +* par écrit. +* +* This file is part of the Ce fichier fait partie du projet +* OpenCADC project. OpenCADC. +* +* OpenCADC is free software: OpenCADC est un logiciel libre ; +* you can redistribute it and/or vous pouvez le redistribuer ou le +* modify it under the terms of modifier suivant les termes de +* the GNU Affero General Public la “GNU Affero General Public +* License as published by the License” telle que publiée +* Free Software Foundation, par la Free Software Foundation +* either version 3 of the : soit la version 3 de cette +* License, or (at your option) licence, soit (à votre gré) +* any later version. toute version ultérieure. +* +* OpenCADC is distributed in the OpenCADC est distribué +* hope that it will be useful, dans l’espoir qu’il vous +* but WITHOUT ANY WARRANTY; sera utile, mais SANS AUCUNE +* without even the implied GARANTIE : sans même la garantie +* warranty of MERCHANTABILITY implicite de COMMERCIALISABILITÉ +* or FITNESS FOR A PARTICULAR ni d’ADÉQUATION À UN OBJECTIF +* PURPOSE. See the GNU Affero PARTICULIER. Consultez la Licence +* General Public License for Générale Publique GNU Affero +* more details. pour plus de détails. +* +* You should have received Vous devriez avoir reçu une +* a copy of the GNU Affero copie de la Licence Générale +* General Public License along Publique GNU Affero avec +* with OpenCADC. If not, see OpenCADC ; si ce n’est +* . pas le cas, consultez : +* . +* +************************************************************************ +*/ + +package org.opencadc.vault; + +import ca.nrc.cadc.auth.AuthenticationUtil; +import ca.nrc.cadc.auth.IdentityManager; +import ca.nrc.cadc.auth.PrincipalExtractor; +import ca.nrc.cadc.auth.X509CertificateChain; +import ca.nrc.cadc.date.DateUtil; +import ca.nrc.cadc.db.TransactionManager; +import ca.nrc.cadc.io.ResourceIterator; +import ca.nrc.cadc.net.TransientException; +import ca.nrc.cadc.util.InvalidConfigException; +import ca.nrc.cadc.util.MultiValuedProperties; +import java.io.IOException; +import java.net.URI; +import java.security.Principal; +import java.text.DateFormat; +import java.util.HashSet; +import java.util.Iterator; +import java.util.Map; +import java.util.Set; +import java.util.TreeMap; +import java.util.TreeSet; +import java.util.UUID; +import javax.security.auth.Subject; +import javax.security.auth.x500.X500Principal; +import org.apache.log4j.Logger; +import org.opencadc.inventory.Artifact; +import org.opencadc.inventory.DeletedArtifactEvent; +import org.opencadc.inventory.Namespace; +import org.opencadc.inventory.db.ArtifactDAO; +import org.opencadc.inventory.db.DeletedArtifactEventDAO; +import org.opencadc.inventory.db.SQLGenerator; +import org.opencadc.vospace.ContainerNode; +import org.opencadc.vospace.DataNode; +import org.opencadc.vospace.LinkNode; +import org.opencadc.vospace.Node; +import org.opencadc.vospace.NodeNotSupportedException; +import org.opencadc.vospace.NodeProperty; +import org.opencadc.vospace.VOS; +import org.opencadc.vospace.db.NodeDAO; +import org.opencadc.vospace.server.NodePersistence; + +/** + * + * @author pdowler + */ +public class NodePersistenceImpl implements NodePersistence { + private static final Logger log = Logger.getLogger(NodePersistenceImpl.class); + + private final Map nodeDaoConfig = new TreeMap<>(); + private final ContainerNode root; + private final ContainerNode trash; + private final Namespace storageNamespace; + private final Set immutableProps = new TreeSet<>(); + private final Set artifactProps = new TreeSet<>(); + private URI resourceID; + + public NodePersistenceImpl(URI resourceID) { + if (resourceID == null) { + throw new IllegalArgumentException("resource ID required"); + } + this.resourceID = resourceID; + MultiValuedProperties config = VaultInitAction.getConfig(); + String dataSourceName = VaultInitAction.JNDI_DATASOURCE; + String inventorySchema = config.getFirstPropertyValue(VaultInitAction.INVENTORY_SCHEMA_KEY); + String vospaceSchema = config.getFirstPropertyValue(VaultInitAction.VOSPACE_SCHEMA_KEY); + nodeDaoConfig.put(SQLGenerator.class.getName(), SQLGenerator.class); + nodeDaoConfig.put("jndiDataSourceName", dataSourceName); + nodeDaoConfig.put("schema", inventorySchema); + nodeDaoConfig.put("vosSchema", vospaceSchema); + + final String owner = config.getFirstPropertyValue(VaultInitAction.ROOT_OWNER); + if (owner == null) { + throw new InvalidConfigException(VaultInitAction.ROOT_OWNER + " cannot be null"); + } + Subject rawOwnerSubject = AuthenticationUtil.getSubject(new PrincipalExtractor() { + @Override + public Set getPrincipals() { + Set ret = new HashSet<>(); + ret.add(new X500Principal(owner)); + return ret; + } + + @Override + public X509CertificateChain getCertificateChain() { + return null; + } + }); + IdentityManager identityManager = AuthenticationUtil.getIdentityManager(); + + // root node + UUID rootID = new UUID(0L, 0L); + this.root = new ContainerNode(rootID, ""); + root.owner = identityManager.augment(rawOwnerSubject); + root.ownerDisplay = identityManager.toDisplayString(root.owner); + log.warn("ROOT owner: " + root.owner); + root.ownerID = identityManager.toOwner(rawOwnerSubject); + root.isPublic = true; + root.inheritPermissions = false; + + // trash node + // TODO: do this setup in a txn with a lock on something + NodeDAO dao = getDAO(); + ContainerNode tn = (ContainerNode) dao.get(root, ".trash"); + if (tn == null) { + tn = new ContainerNode(".trash"); + } + // always reset props to current config + tn.ownerID = root.ownerID; + tn.owner = root.owner; + tn.isPublic = false; + tn.inheritPermissions = false; + tn.parentID = rootID; + dao.put(tn); + this.trash = tn; + + String ns = config.getFirstPropertyValue(VaultInitAction.STORAGE_NAMESPACE_KEY); + this.storageNamespace = new Namespace(ns); + + // computed properties + // VOS.PROPERTY_URI_AVAILABLESPACE // container nodes + // VOS.PROPERTY_URI_WRITABLE // prediction for current caller + + // props only the admin (root owner) can modify? + // VOS.PROPERTY_URI_CREATOR // owner + // VOS.PROPERTY_URI_CONTENTLENGTH // container nodes + // VOS.PROPERTY_URI_QUOTA // container nodes + + // node properties that match immutable Artifact fields + immutableProps.add(VOS.PROPERTY_URI_CONTENTLENGTH); // immutable + immutableProps.add(VOS.PROPERTY_URI_CONTENTMD5); // immutable + immutableProps.add(VOS.PROPERTY_URI_CREATION_DATE); // immutable + + artifactProps.addAll(immutableProps); + artifactProps.add(VOS.PROPERTY_URI_CONTENTENCODING); // mutable + artifactProps.add(VOS.PROPERTY_URI_TYPE); // mutable + } + + private NodeDAO getDAO() { + NodeDAO instance = new NodeDAO(); + instance.setConfig(nodeDaoConfig); + return instance; + } + + private ArtifactDAO getArtifactDAO() { + ArtifactDAO instance = new ArtifactDAO(true); // origin==true? + instance.setConfig(nodeDaoConfig); + return instance; + } + + private URI generateStorageID() { + UUID id = UUID.randomUUID(); + URI ret = URI.create(storageNamespace.getNamespace() + id.toString()); + return ret; + } + + @Override + public URI getResourceID() { + return resourceID; + } + + /** + * Get the container node that represents the root of all other nodes. + * This container node is used to navigate a path (from the root) using + * get(ContainerNode parent, String name). + * + * @return the root container node + */ + @Override + public ContainerNode getRootNode() { + return root; + } + + @Override + public Set getImmutableProps() { + return immutableProps; + } + + /** + * Get a node by name. Concept: The caller uses this to navigate the path + * from the root node to the target, checking permissions and deciding what + * to do about LinkNode(s) along the way. + * + * @param parent parent node, may be special root node but not null + * @param name relative name of the child node + * @return the child node or null if it does not exist + * @throws TransientException + */ + @Override + public Node get(ContainerNode parent, String name) throws TransientException { + if (parent == null || name == null) { + throw new IllegalArgumentException("args cannot be null: parent, name"); + } + NodeDAO dao = getDAO(); + Node ret = dao.get(parent, name); + if (ret == null) { + return null; + } + ret.parent = parent; + IdentityManager identityManager = AuthenticationUtil.getIdentityManager(); + ret.owner = identityManager.toSubject(ret.ownerID); + ret.ownerDisplay = identityManager.toDisplayString(ret.owner); + + // in principle we could have queried vospace.Node join inventory.Artifact above + // and avoid this query.... simplicity for now + if (ret instanceof DataNode) { + DataNode dn = (DataNode) ret; + ArtifactDAO artifactDAO = getArtifactDAO(); + Artifact a = artifactDAO.get(dn.storageID); + if (a != null) { + DateFormat df = DateUtil.getDateFormat(DateUtil.IVOA_DATE_FORMAT, DateUtil.UTC); + ret.getProperties().add(new NodeProperty(VOS.PROPERTY_URI_CONTENTLENGTH, a.getContentLength().toString())); + // assume MD5 + ret.getProperties().add(new NodeProperty(VOS.PROPERTY_URI_CONTENTMD5, a.getContentChecksum().getSchemeSpecificPart())); + ret.getProperties().add(new NodeProperty(VOS.PROPERTY_URI_DATE, df.format(a.getContentLastModified()))); + if (a.contentEncoding != null) { + ret.getProperties().add(new NodeProperty(VOS.PROPERTY_URI_CONTENTENCODING, a.contentEncoding)); + } + if (a.contentType != null) { + ret.getProperties().add(new NodeProperty(VOS.PROPERTY_URI_TYPE, a.contentType)); + } + } + } + return ret; + } + + /** + * Get an iterator over the children of a node. The output can optionally be + * limited to a specific number of children and can optionally start at a + * specific child (usually the last one from a previous "batch") to resume + * listing at a known position. + * + * @param parent the container to iterate + * @param limit max number of nodes to return, may be null + * @param start first node in order to consider, may be null + * @return iterator of matching child nodes, may be empty + */ + @Override + public ResourceIterator iterator(ContainerNode parent, Integer limit, String start) { + if (parent == null) { + throw new IllegalArgumentException("arg cannot be null: parent"); + } + NodeDAO dao = getDAO(); + ResourceIterator ret = dao.iterator(parent, limit, start); + return new IdentWrapper(parent, ret); + } + + private class IdentWrapper implements ResourceIterator { + + private final ContainerNode parent; + private final ResourceIterator childIter; + + private IdentityManager identityManager = AuthenticationUtil.getIdentityManager(); + private Map identCache = new TreeMap<>(); + + IdentWrapper(ContainerNode parent, ResourceIterator childIter) { + this.parent = parent; + this.childIter = childIter; + // prime cache with caller + Subject caller = AuthenticationUtil.getCurrentSubject(); + if (caller != null) { + Object ownerID = identityManager.toOwner(caller); + if (ownerID != null) { + // HACK: NodeDAO returns ownerID as String and relies on the IM + // to convert to a number (eg) + identCache.put(ownerID.toString(), caller); + } + } + } + + @Override + public boolean hasNext() { + return childIter.hasNext(); + } + + @Override + public Node next() { + Node ret = childIter.next(); + ret.parent = parent; + Subject s = identCache.get(ret.ownerID); + if (s == null) { + s = identityManager.toSubject(ret.ownerID); + identCache.put(ret.ownerID, s); + } + ret.owner = s; + ret.ownerDisplay = identityManager.toDisplayString(ret.owner); + return ret; + } + + @Override + public void close() throws IOException { + childIter.close(); + identCache.clear(); + } + + } + + /** + * Load additional node properties for the specified node. Note: this may not be + * necessary and may be removed. TBD. + * + * @param node + * @throws TransientException + */ + @Override + public void getProperties(Node node) throws TransientException { + // no-op + } + + /** + * Put the specified node. This can be an insert or update; to update, the argument + * node must have been retrieved from persistence so it has the right Entity.id + * value. This method may modify the Entity.metaChecksum and the Entity.lastModified + * values. + * + * @param node the node to insert or update + * @return the possibly modified node + * @throws NodeNotSupportedException + * @throws TransientException + */ + @Override + public Node put(Node node) throws NodeNotSupportedException, TransientException { + if (node == null) { + throw new IllegalArgumentException("arg cannot be null: node"); + } + if (node.parentID == null) { + if (node.parent == null) { + throw new RuntimeException("BUG: cannot persist node without parent: " + node); + } + node.parentID = node.parent.getID(); + } + if (node.ownerID == null) { + if (node.owner == null) { + throw new RuntimeException("BUG: cannot persist node without owner: " + node); + } + IdentityManager identityManager = AuthenticationUtil.getIdentityManager(); + node.ownerID = identityManager.toOwner(node.owner); + } + + if (node instanceof DataNode) { + DataNode dn = (DataNode) node; + if (dn.storageID == null) { + // new data node? if lastModified is assigned, this looks sketchy + if (dn.getLastModified() != null) { + throw new RuntimeException( + "BUG: attempt to put a previously stored DataNode without persistent storageID: " + + dn.getID() + " aka " + dn); + } + // concept: use a persistent storageID in the node that resolves to a a file + // once someone puts the file to minoc, so Node.storageID == Artifact.uri + // but the artifact may or may not exist + dn.storageID = generateStorageID(); + + } else { + // update existing data node + // need to remove all artifact props from the node.getProperties() + // and use artifactDAO to set the mutable ones + NodeProperty contentType = null; + NodeProperty contentEncoding = null; + + Iterator i = dn.getProperties().iterator(); + while (i.hasNext()) { + NodeProperty np = i.next(); + if (VOS.PROPERTY_URI_TYPE.equals(np.getKey())) { + contentType = np; + } else if (VOS.PROPERTY_URI_CONTENTENCODING.equals(np.getKey())) { + contentEncoding = np; + } + + if (artifactProps.contains(np.getKey())) { + i.remove(); + } + } + if (contentType != null || contentEncoding != null) { // optimization + ArtifactDAO artifactDAO = getArtifactDAO(); + Artifact a = artifactDAO.get(dn.storageID); + if (a != null) { + if (contentType != null) { + if (contentType.isMarkedForDeletion()) { + a.contentType = null; + } else { + a.contentType = contentType.getValue(); + } + } + if (contentEncoding != null) { + if (contentEncoding.isMarkedForDeletion()) { + a.contentEncoding = null; + } else { + a.contentEncoding = contentEncoding.getValue(); + } + } + artifactDAO.put(a); + } + } + } + } + + NodeDAO dao = getDAO(); + dao.put(node); + return node; + } + + /** + * Delete the specified node. + * + * @param node the node to delete + * @throws TransientException + */ + @Override + public void delete(Node node) throws TransientException { + if (node == null) { + throw new IllegalArgumentException("arg cannot be null: node"); + } + + final NodeDAO dao = getDAO(); + final ArtifactDAO artifactDAO = getArtifactDAO(); + TransactionManager txn = dao.getTransactionManager(); + + try { + log.debug("starting transaction"); + txn.startTransaction(); + log.debug("start txn: OK"); + + Node locked = dao.lock(node); + if (locked != null) { + node = locked; // safer than having two vars and accidentally using the wrong one + URI storageID = null; + if (node instanceof ContainerNode) { + ContainerNode cn = (ContainerNode) node; + boolean empty = dao.isEmpty(cn); + if (!empty) { + log.debug("commit txn..."); + txn.commitTransaction(); + log.debug("commit txn: OK"); + throw new IllegalArgumentException("container node '" + node.getName() + "' is not empty"); + } + } else if (node instanceof DataNode) { + DataNode dn = (DataNode) node; + NodeProperty len = dn.getProperty(VOS.PROPERTY_URI_CONTENTLENGTH); + if (len != null) { + // artifact exists + storageID = dn.storageID; + } + } // else: LinkNode can always be deleted + + if (storageID != null) { + Artifact a = artifactDAO.get(storageID); + if (a != null) { + DeletedArtifactEventDAO daeDAO = new DeletedArtifactEventDAO(artifactDAO); + DeletedArtifactEvent dae = new DeletedArtifactEvent(a.getID()); + daeDAO.put(dae); + artifactDAO.delete(a.getID()); + } + } + // TODO: need DeletedNodeDAO to create DeletedNodeEvent + dao.delete(node.getID()); + } else { + log.debug("failed to lock node " + node.getID() + " - assume deleted by another process"); + } + + log.debug("commit txn..."); + txn.commitTransaction(); + log.debug("commit txn: OK"); + } catch (Exception ex) { + if (txn.isOpen()) { + log.error("failed to delete " + node.getID() + " aka " + node.getName(), ex); + txn.rollbackTransaction(); + log.debug("rollback txn: OK"); + } + throw ex; + } finally { + if (txn.isOpen()) { + log.error("BUG - open transaction in finally"); + txn.rollbackTransaction(); + log.error("rollback txn: OK"); + } + } + } +} diff --git a/vault/src/main/java/org/opencadc/vault/VaultInitAction.java b/vault/src/main/java/org/opencadc/vault/VaultInitAction.java index 2f6a34f71..c22083be8 100644 --- a/vault/src/main/java/org/opencadc/vault/VaultInitAction.java +++ b/vault/src/main/java/org/opencadc/vault/VaultInitAction.java @@ -63,7 +63,7 @@ * . * ************************************************************************ -*/ + */ package org.opencadc.vault; @@ -71,30 +71,47 @@ import ca.nrc.cadc.rest.InitAction; import ca.nrc.cadc.util.MultiValuedProperties; import ca.nrc.cadc.util.PropertiesReader; +import ca.nrc.cadc.uws.server.impl.InitDatabaseUWS; import java.net.URI; import java.net.URISyntaxException; import java.util.Map; import java.util.TreeMap; +import javax.naming.Context; +import javax.naming.InitialContext; +import javax.naming.NamingException; import javax.sql.DataSource; import org.apache.log4j.Logger; +import org.opencadc.inventory.Namespace; +import org.opencadc.vospace.db.InitDatabaseVOS; +import org.opencadc.vospace.server.NodePersistence; /** * * @author pdowler */ public class VaultInitAction extends InitAction { + private static final Logger log = Logger.getLogger(VaultInitAction.class); - + static final String JNDI_DATASOURCE = "jdbc/nodes"; // context.xml - + static final String JNDI_UWS_DATASOURCE = "jdbc/uws"; // context.xml + // config keys private static final String VAULT_KEY = "org.opencadc.vault"; static final String RESOURCE_ID_KEY = VAULT_KEY + ".resourceID"; - static final String SCHEMA_KEY = VAULT_KEY + ".nodes.schema"; - + static final String INVENTORY_SCHEMA_KEY = VAULT_KEY + ".inventory.schema"; + static final String VOSPACE_SCHEMA_KEY = VAULT_KEY + ".vospace.schema"; + + static final String ROOT_OWNER = VAULT_KEY + ".root.owner"; // numeric? + + static final String STORAGE_NAMESPACE_KEY = VAULT_KEY + ".storage.namespace"; + MultiValuedProperties props; private URI resourceID; - private Map daoConfig; + private Namespace storageNamespace; + private Map daoConfig; + + private String jndiNodePersistence; public VaultInitAction() { super(); @@ -104,11 +121,13 @@ public VaultInitAction() { public void doInit() { initConfig(); initDatabase(); + initUWSDatabase(); + initNodePersistence(); } - + /** * Read config file and verify that all required entries are present. - * + * * @return MultiValuedProperties containing the application config * @throws IllegalStateException if required config items are missing */ @@ -129,9 +148,27 @@ static MultiValuedProperties getConfig() { sb.append("OK"); } - String schema = mvp.getFirstPropertyValue(SCHEMA_KEY); - sb.append("\n\t").append(SCHEMA_KEY).append(": "); - if (schema == null) { + String invSchema = mvp.getFirstPropertyValue(INVENTORY_SCHEMA_KEY); + sb.append("\n\t").append(INVENTORY_SCHEMA_KEY).append(": "); + if (invSchema == null) { + sb.append("MISSING"); + ok = false; + } else { + sb.append("OK"); + } + + String vosSchema = mvp.getFirstPropertyValue(VOSPACE_SCHEMA_KEY); + sb.append("\n\t").append(VOSPACE_SCHEMA_KEY).append(": "); + if (vosSchema == null) { + sb.append("MISSING"); + ok = false; + } else { + sb.append("OK"); + } + + String ns = mvp.getFirstPropertyValue(STORAGE_NAMESPACE_KEY); + sb.append("\n\t").append(STORAGE_NAMESPACE_KEY).append(": "); + if (ns == null) { sb.append("MISSING"); ok = false; } else { @@ -144,40 +181,83 @@ static MultiValuedProperties getConfig() { return mvp; } - - static Map getDaoConfig(MultiValuedProperties props) { + + static Map getDaoConfig(MultiValuedProperties props) { Map ret = new TreeMap<>(); ret.put("jndiDataSourceName", org.opencadc.vault.VaultInitAction.JNDI_DATASOURCE); - ret.put("schema", props.getFirstPropertyValue(org.opencadc.vault.VaultInitAction.SCHEMA_KEY)); + ret.put("schema", props.getFirstPropertyValue(org.opencadc.vault.VaultInitAction.INVENTORY_SCHEMA_KEY)); + ret.put("vosSchema", props.getFirstPropertyValue(org.opencadc.vault.VaultInitAction.VOSPACE_SCHEMA_KEY)); return ret; } - + private void initConfig() { log.info("initConfig: START"); this.props = getConfig(); String rid = props.getFirstPropertyValue(RESOURCE_ID_KEY); - + String ns = props.getFirstPropertyValue(STORAGE_NAMESPACE_KEY); try { this.resourceID = new URI(rid); + this.storageNamespace = new Namespace(ns); this.daoConfig = getDaoConfig(props); log.info("initConfig: OK"); } catch (URISyntaxException ex) { throw new IllegalStateException("invalid config: " + RESOURCE_ID_KEY + " must be a valid URI"); } } - + private void initDatabase() { log.info("initDatabase: START"); try { DataSource ds = DBUtil.findJNDIDataSource(JNDI_DATASOURCE); - String database = (String) daoConfig.get("database"); - String schema = (String) daoConfig.get("schema"); - //VaultInitDatabase init = new VaultInitDatabase(ds, database, schema); - //init.doInit(); + String schema = (String) daoConfig.get("vosSchema"); + InitDatabaseVOS init = new InitDatabaseVOS(ds, null, schema); + init.doInit(); log.info("initDatabase: " + JNDI_DATASOURCE + " " + schema + " OK"); } catch (Exception ex) { throw new IllegalStateException("check/init database failed", ex); } } + private void initUWSDatabase() { + log.info("initUWSDatabase: START"); + try { + // Init UWS database + DataSource uws = DBUtil.findJNDIDataSource(JNDI_UWS_DATASOURCE); + InitDatabaseUWS uwsi = new InitDatabaseUWS(uws, null, "uws"); + uwsi.doInit(); + log.info("initDatabase: " + JNDI_UWS_DATASOURCE + " uws OK"); + + } catch (Exception ex) { + throw new RuntimeException("check/init uws database failed", ex); + } + } + + protected void initNodePersistence() { + jndiNodePersistence = componentID + ".nodePersistence"; + try { + Context ctx = new InitialContext(); + try { + ctx.unbind(jndiNodePersistence); + } catch (NamingException ignore) { + log.debug("unbind previous JNDI key (" + jndiNodePersistence + ") failed... ignoring"); + } + NodePersistence npi = new NodePersistenceImpl(resourceID); + ctx.bind(jndiNodePersistence, npi); + + log.info("created JNDI key: " + jndiNodePersistence + " impl: " + npi.getClass().getName()); + } catch (Exception ex) { + log.error("Failed to create JNDI Key " + jndiNodePersistence, ex); + } + } + + @Override + public void doShutdown() { + try { + Context ctx = new InitialContext(); + ctx.unbind(jndiNodePersistence); + } catch (Exception oops) { + log.error("unbind failed during destroy", oops); + } + } + } diff --git a/vault/src/main/webapp/META-INF/context.xml b/vault/src/main/webapp/META-INF/context.xml index cbc72f2e0..e5775723b 100644 --- a/vault/src/main/webapp/META-INF/context.xml +++ b/vault/src/main/webapp/META-INF/context.xml @@ -15,4 +15,17 @@ removeAbandoned="false" testOnBorrow="true" validationQuery="select 123" /> + + + diff --git a/vault/src/main/webapp/WEB-INF/web.xml b/vault/src/main/webapp/WEB-INF/web.xml index 2d519aa70..90476792e 100644 --- a/vault/src/main/webapp/WEB-INF/web.xml +++ b/vault/src/main/webapp/WEB-INF/web.xml @@ -5,7 +5,6 @@ "http://java.sun.com/j2ee/dtds/web-app_2_3.dtd"> - vault @@ -20,9 +19,11 @@ org.opencadc.vault ca.nrc.cadc.db + org.opencadc.vospace ca.nrc.cadc.rest ca.nrc.cadc.util ca.nrc.cadc.vosi + ca.nrc.cadc.uws @@ -34,51 +35,95 @@ NodesServlet - ca.nrc.cadc.rest.RestServlet - - init - org.opencadc.vault.VaultInitAction - - 2 - - - - - CapabilitiesServlet - ca.nrc.cadc.rest.RestServlet - - init - ca.nrc.cadc.vosi.CapInitAction - - - head - ca.nrc.cadc.vosi.CapHeadAction - - - get - ca.nrc.cadc.vosi.CapGetAction - - - input - /capabilities.xml - - 3 - - - - - AvailabilityServlet - ca.nrc.cadc.vosi.AvailabilityServlet - - ca.nrc.cadc.vosi.AvailabilityPlugin - org.opencadc.vault.ServiceAvailability - - - availabilityProperties - vault-availability.properties - - 4 - + ca.nrc.cadc.rest.RestServlet + + augmentSubject + true + + + init + org.opencadc.vault.VaultInitAction + + + get + org.opencadc.vospace.server.actions.GetNodeAction + + + put + org.opencadc.vospace.server.actions.CreateNodeAction + + + post + org.opencadc.vospace.server.actions.UpdateNodeAction + + + delete + org.opencadc.vospace.server.actions.DeleteNodeAction + + 2 + + + + + RecursiveDeleteNodeServlet + ca.nrc.cadc.uws.server.JobServlet + + get + ca.nrc.cadc.uws.web.GetAction + + + post + ca.nrc.cadc.uws.web.PostAction + + + delete + ca.nrc.cadc.uws.web.DeleteAction + + + ca.nrc.cadc.uws.server.JobManager + org.opencadc.vault.RecursiveDeleteNodeJobManager + + 3 + + + + + + CapabilitiesServlet + ca.nrc.cadc.rest.RestServlet + + init + ca.nrc.cadc.vosi.CapInitAction + + + head + ca.nrc.cadc.vosi.CapHeadAction + + + get + ca.nrc.cadc.vosi.CapGetAction + + + input + /capabilities.xml + + 3 + + + + + AvailabilityServlet + ca.nrc.cadc.vosi.AvailabilityServlet + + ca.nrc.cadc.vosi.AvailabilityPlugin + org.opencadc.vault.ServiceAvailability + + + availabilityProperties + vault-availability.properties + + 4 + @@ -86,7 +131,12 @@ NodesServlet /nodes/* - + + + RecursiveDeleteNodeServlet + /recursiveDelete/* + + AvailabilityServlet @@ -105,4 +155,9 @@ /logControl + + index.html + + + diff --git a/vault/src/main/webapp/capabilities.xml b/vault/src/main/webapp/capabilities.xml index 0a6205003..d24f1a220 100644 --- a/vault/src/main/webapp/capabilities.xml +++ b/vault/src/main/webapp/capabilities.xml @@ -6,23 +6,146 @@ - https://replace.com/vault/capabilities + https://replace.me.com/vault/capabilities - https://replace.com/vault/availability - - + https://replace.me.com/vault/availability - https://replace.com/vault/logControl + https://replace.me.com/vault/logControl - + + + + + + https://replace.me.com/vault/nodes + + + + + + + + + + https://replace.me.com/vault/recursiveDelete + + + + + + + + + + + + + + + + + + + + + + + +