From e382a56cef3952dc15517256dac02b1c5df82317 Mon Sep 17 00:00:00 2001 From: Daniel Roberts Date: Thu, 25 Jul 2024 00:24:42 +0000 Subject: [PATCH 1/2] Handle Fate table creation on upgrade Upgrader now calls the same code for fate table creation as the init code --- .../server/init/FileSystemInitializer.java | 40 ++++++------------- .../manager/upgrade/Upgrader12to13.java | 32 +++++++++++---- 2 files changed, 36 insertions(+), 36 deletions(-) diff --git a/server/base/src/main/java/org/apache/accumulo/server/init/FileSystemInitializer.java b/server/base/src/main/java/org/apache/accumulo/server/init/FileSystemInitializer.java index 7012fa93b7e..3d8d5d2c037 100644 --- a/server/base/src/main/java/org/apache/accumulo/server/init/FileSystemInitializer.java +++ b/server/base/src/main/java/org/apache/accumulo/server/init/FileSystemInitializer.java @@ -18,21 +18,16 @@ */ package org.apache.accumulo.server.init; -import static org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.ServerColumnFamily.DIRECTORY_COLUMN; -import static org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.ServerColumnFamily.TIME_COLUMN; -import static org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.TabletColumnFamily.AVAILABILITY_COLUMN; -import static org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.TabletColumnFamily.PREV_ROW_COLUMN; - import java.io.FileNotFoundException; import java.io.IOException; import java.util.HashMap; import java.util.Map; import java.util.TreeMap; +import java.util.stream.Collectors; import org.apache.accumulo.core.Constants; import org.apache.accumulo.core.client.admin.TabletAvailability; import org.apache.accumulo.core.client.admin.TimeType; -import org.apache.accumulo.core.clientImpl.TabletAvailabilityUtil; import org.apache.accumulo.core.conf.AccumuloConfiguration; import org.apache.accumulo.core.conf.DefaultConfiguration; import org.apache.accumulo.core.crypto.CryptoFactoryLoader; @@ -40,6 +35,7 @@ import org.apache.accumulo.core.data.Mutation; import org.apache.accumulo.core.data.TableId; import org.apache.accumulo.core.data.Value; +import org.apache.accumulo.core.dataImpl.KeyExtent; import org.apache.accumulo.core.file.FileOperations; import org.apache.accumulo.core.file.FileSKVWriter; import org.apache.accumulo.core.metadata.AccumuloTable; @@ -48,9 +44,9 @@ import org.apache.accumulo.core.metadata.schema.DataFileValue; import org.apache.accumulo.core.metadata.schema.MetadataSchema; import org.apache.accumulo.core.metadata.schema.MetadataTime; +import org.apache.accumulo.core.metadata.schema.TabletMetadata; import org.apache.accumulo.core.spi.crypto.CryptoService; import org.apache.accumulo.core.spi.fs.VolumeChooserEnvironment; -import org.apache.accumulo.core.util.ColumnFQ; import org.apache.accumulo.server.ServerContext; import org.apache.accumulo.server.conf.store.TablePropKey; import org.apache.accumulo.server.fs.VolumeChooserEnvironmentImpl; @@ -91,28 +87,16 @@ public static class InitialTablet { this.extent = new Text(MetadataSchema.TabletsSection.encodeRow(this.tableId, this.endRow)); } - private TreeMap createEntries() { - TreeMap sorted = new TreeMap<>(); - Value EMPTY_SIZE = new DataFileValue(0, 0).encodeAsValue(); - sorted.put(new Key(this.extent, DIRECTORY_COLUMN.getColumnFamily(), - DIRECTORY_COLUMN.getColumnQualifier(), 0), new Value(this.dirName)); - sorted.put( - new Key(this.extent, TIME_COLUMN.getColumnFamily(), TIME_COLUMN.getColumnQualifier(), 0), - new Value(new MetadataTime(0, TimeType.LOGICAL).encode())); - sorted.put( - new Key(this.extent, PREV_ROW_COLUMN.getColumnFamily(), - PREV_ROW_COLUMN.getColumnQualifier(), 0), - MetadataSchema.TabletsSection.TabletColumnFamily.encodePrevEndRow(this.prevEndRow)); - sorted.put( - new Key(this.extent, AVAILABILITY_COLUMN.getColumnFamily(), - AVAILABILITY_COLUMN.getColumnQualifier(), 0), - TabletAvailabilityUtil.toValue(TabletAvailability.HOSTED)); - for (String file : this.files) { - var col = - new ColumnFQ(MetadataSchema.TabletsSection.DataFileColumnFamily.NAME, new Text(file)); - sorted.put(new Key(extent, col.getColumnFamily(), col.getColumnQualifier(), 0), EMPTY_SIZE); + private Map createEntries() { + KeyExtent keyExtent = new KeyExtent(tableId, endRow, prevEndRow); + var builder = TabletMetadata.builder(keyExtent).putDirName(dirName) + .putTime(new MetadataTime(0, TimeType.LOGICAL)) + .putTabletAvailability(TabletAvailability.HOSTED).putPrevEndRow(prevEndRow); + for (String file : files) { + builder.putFile(new ReferencedTabletFile(new Path(file)).insert(), new DataFileValue(0, 0)); } - return sorted; + return builder.build().getKeyValues().stream() + .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); } public Mutation createMutation() { diff --git a/server/manager/src/main/java/org/apache/accumulo/manager/upgrade/Upgrader12to13.java b/server/manager/src/main/java/org/apache/accumulo/manager/upgrade/Upgrader12to13.java index 6d89ebc8ca4..825a5ebd990 100644 --- a/server/manager/src/main/java/org/apache/accumulo/manager/upgrade/Upgrader12to13.java +++ b/server/manager/src/main/java/org/apache/accumulo/manager/upgrade/Upgrader12to13.java @@ -23,21 +23,23 @@ import static org.apache.accumulo.core.metadata.schema.MetadataSchema.RESERVED_PREFIX; import static org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.Upgrade12to13.COMPACT_COL; +import java.io.IOException; import java.util.ArrayList; import java.util.List; import java.util.Map; import java.util.Map.Entry; import org.apache.accumulo.core.Constants; +import org.apache.accumulo.core.client.BatchWriter; +import org.apache.accumulo.core.client.MutationsRejectedException; +import org.apache.accumulo.core.client.TableNotFoundException; import org.apache.accumulo.core.client.admin.TabletAvailability; -import org.apache.accumulo.core.clientImpl.Namespace; import org.apache.accumulo.core.conf.Property; import org.apache.accumulo.core.data.Key; import org.apache.accumulo.core.data.Mutation; import org.apache.accumulo.core.data.TableId; import org.apache.accumulo.core.data.Value; import org.apache.accumulo.core.fate.zookeeper.ZooUtil; -import org.apache.accumulo.core.manager.state.tables.TableState; import org.apache.accumulo.core.metadata.AccumuloTable; import org.apache.accumulo.core.metadata.schema.Ample.DataLevel; import org.apache.accumulo.core.metadata.schema.Ample.TabletsMutator; @@ -51,7 +53,9 @@ import org.apache.accumulo.core.security.Authorizations; import org.apache.accumulo.server.ServerContext; import org.apache.accumulo.server.conf.store.TablePropKey; -import org.apache.accumulo.server.tables.TableManager; +import org.apache.accumulo.server.init.FileSystemInitializer; +import org.apache.accumulo.server.init.InitialConfiguration; +import org.apache.accumulo.server.init.ZooKeeperInitializer; import org.apache.accumulo.server.util.PropUtil; import org.apache.zookeeper.KeeperException; import org.apache.zookeeper.data.Stat; @@ -120,13 +124,25 @@ private static void addCompactionsNode(ServerContext context) { } private void createFateTable(ServerContext context) { + ZooKeeperInitializer zkInit = new ZooKeeperInitializer(); + zkInit.initFateTableState(context); + try { - TableManager.prepareNewTableState(context, AccumuloTable.FATE.tableId(), - Namespace.ACCUMULO.id(), AccumuloTable.FATE.tableName(), TableState.ONLINE, - ZooUtil.NodeExistsPolicy.SKIP); - } catch (KeeperException | InterruptedException e) { - throw new IllegalStateException("Error creating table: " + AccumuloTable.FATE.tableName(), e); + FileSystemInitializer initializer = new FileSystemInitializer( + new InitialConfiguration(context.getHadoopConf(), context.getSiteConfiguration())); + FileSystemInitializer.InitialTablet fateTableTableTablet = + initializer.createFateRefTablet(context); + // Add references to the Metadata Table + try (BatchWriter writer = context.createBatchWriter(AccumuloTable.METADATA.tableName())) { + writer.addMutation(fateTableTableTablet.createMutation()); + } catch (MutationsRejectedException | TableNotFoundException e) { + LOG.error("Failed to write tablet refs to metadata table"); + throw new RuntimeException(e); + } + } catch (IOException e) { + LOG.error("Problem attempting to create Fate table", e); } + LOG.info("Created Fate table"); } private void removeCompactColumnsFromRootTabletMetadata(ServerContext context) { From da8be8b4d528291ccf895ed6d3adbb6b460710b0 Mon Sep 17 00:00:00 2001 From: Daniel Roberts Date: Thu, 25 Jul 2024 18:55:58 +0000 Subject: [PATCH 2/2] Use the metadata path vs the metadata --- .../org/apache/accumulo/server/init/FileSystemInitializer.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/base/src/main/java/org/apache/accumulo/server/init/FileSystemInitializer.java b/server/base/src/main/java/org/apache/accumulo/server/init/FileSystemInitializer.java index 3d8d5d2c037..066a48161c4 100644 --- a/server/base/src/main/java/org/apache/accumulo/server/init/FileSystemInitializer.java +++ b/server/base/src/main/java/org/apache/accumulo/server/init/FileSystemInitializer.java @@ -159,7 +159,7 @@ void initialize(VolumeManager fs, String rootTabletDirUri, String rootTabletFile // populate the root tablet with info about the metadata table's two initial tablets InitialTablet tablesTablet = new InitialTablet(AccumuloTable.METADATA.tableId(), TABLE_TABLETS_TABLET_DIR, null, - SPLIT_POINT, StoredTabletFile.of(new Path(metadataFileName)).getMetadata()); + SPLIT_POINT, StoredTabletFile.of(new Path(metadataFileName)).getMetadataPath()); InitialTablet defaultTablet = new InitialTablet(AccumuloTable.METADATA.tableId(), defaultMetadataTabletDirName, SPLIT_POINT, null); createMetadataFile(fs, rootTabletFileUri, tablesTablet, defaultTablet);