Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Handle Fate table creation on upgrade #4760

Merged
merged 2 commits into from
Jul 25, 2024
Merged

Handle Fate table creation on upgrade #4760

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
e382a56
Handle Fate table creation on upgrade
ddanielr Jul 25, 2024
da8be8b
Use the metadata path vs the metadata
ddanielr Jul 25, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
42 changes: 13 additions & 29 deletions server/base/src/main/java/org/apache/accumulo/server/init/FileSystemInitializer.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,28 +18,24 @@
*/
package org.apache.accumulo.server.init;

import static org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.ServerColumnFamily.DIRECTORY_COLUMN;
import static org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.ServerColumnFamily.TIME_COLUMN;
import static org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.TabletColumnFamily.AVAILABILITY_COLUMN;
import static org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.TabletColumnFamily.PREV_ROW_COLUMN;

import java.io.FileNotFoundException;
import java.io.IOException;
import java.util.HashMap;
import java.util.Map;
import java.util.TreeMap;
import java.util.stream.Collectors;

import org.apache.accumulo.core.Constants;
import org.apache.accumulo.core.client.admin.TabletAvailability;
import org.apache.accumulo.core.client.admin.TimeType;
import org.apache.accumulo.core.clientImpl.TabletAvailabilityUtil;
import org.apache.accumulo.core.conf.AccumuloConfiguration;
import org.apache.accumulo.core.conf.DefaultConfiguration;
import org.apache.accumulo.core.crypto.CryptoFactoryLoader;
import org.apache.accumulo.core.data.Key;
import org.apache.accumulo.core.data.Mutation;
import org.apache.accumulo.core.data.TableId;
import org.apache.accumulo.core.data.Value;
import org.apache.accumulo.core.dataImpl.KeyExtent;
import org.apache.accumulo.core.file.FileOperations;
import org.apache.accumulo.core.file.FileSKVWriter;
import org.apache.accumulo.core.metadata.AccumuloTable;
Expand All @@ -48,9 +44,9 @@
import org.apache.accumulo.core.metadata.schema.DataFileValue;
import org.apache.accumulo.core.metadata.schema.MetadataSchema;
import org.apache.accumulo.core.metadata.schema.MetadataTime;
import org.apache.accumulo.core.metadata.schema.TabletMetadata;
import org.apache.accumulo.core.spi.crypto.CryptoService;
import org.apache.accumulo.core.spi.fs.VolumeChooserEnvironment;
import org.apache.accumulo.core.util.ColumnFQ;
import org.apache.accumulo.server.ServerContext;
import org.apache.accumulo.server.conf.store.TablePropKey;
import org.apache.accumulo.server.fs.VolumeChooserEnvironmentImpl;
Expand Down Expand Up @@ -91,28 +87,16 @@ public static class InitialTablet {
this.extent = new Text(MetadataSchema.TabletsSection.encodeRow(this.tableId, this.endRow));
}

private TreeMap<Key,Value> createEntries() {
TreeMap<Key,Value> sorted = new TreeMap<>();
Value EMPTY_SIZE = new DataFileValue(0, 0).encodeAsValue();
sorted.put(new Key(this.extent, DIRECTORY_COLUMN.getColumnFamily(),
DIRECTORY_COLUMN.getColumnQualifier(), 0), new Value(this.dirName));
sorted.put(
new Key(this.extent, TIME_COLUMN.getColumnFamily(), TIME_COLUMN.getColumnQualifier(), 0),
new Value(new MetadataTime(0, TimeType.LOGICAL).encode()));
sorted.put(
new Key(this.extent, PREV_ROW_COLUMN.getColumnFamily(),
PREV_ROW_COLUMN.getColumnQualifier(), 0),
MetadataSchema.TabletsSection.TabletColumnFamily.encodePrevEndRow(this.prevEndRow));
sorted.put(
new Key(this.extent, AVAILABILITY_COLUMN.getColumnFamily(),
AVAILABILITY_COLUMN.getColumnQualifier(), 0),
TabletAvailabilityUtil.toValue(TabletAvailability.HOSTED));
for (String file : this.files) {
var col =
new ColumnFQ(MetadataSchema.TabletsSection.DataFileColumnFamily.NAME, new Text(file));
sorted.put(new Key(extent, col.getColumnFamily(), col.getColumnQualifier(), 0), EMPTY_SIZE);
private Map<Key,Value> createEntries() {
KeyExtent keyExtent = new KeyExtent(tableId, endRow, prevEndRow);
var builder = TabletMetadata.builder(keyExtent).putDirName(dirName)
.putTime(new MetadataTime(0, TimeType.LOGICAL))
.putTabletAvailability(TabletAvailability.HOSTED).putPrevEndRow(prevEndRow);
for (String file : files) {
builder.putFile(new ReferencedTabletFile(new Path(file)).insert(), new DataFileValue(0, 0));
}
return sorted;
return builder.build().getKeyValues().stream()
.collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue));
}

public Mutation createMutation() {
Expand Down Expand Up @@ -175,7 +159,7 @@ void initialize(VolumeManager fs, String rootTabletDirUri, String rootTabletFile
// populate the root tablet with info about the metadata table's two initial tablets
InitialTablet tablesTablet =
new InitialTablet(AccumuloTable.METADATA.tableId(), TABLE_TABLETS_TABLET_DIR, null,
SPLIT_POINT, StoredTabletFile.of(new Path(metadataFileName)).getMetadata());
SPLIT_POINT, StoredTabletFile.of(new Path(metadataFileName)).getMetadataPath());
InitialTablet defaultTablet = new InitialTablet(AccumuloTable.METADATA.tableId(),
defaultMetadataTabletDirName, SPLIT_POINT, null);
createMetadataFile(fs, rootTabletFileUri, tablesTablet, defaultTablet);
Expand Down
32 changes: 24 additions & 8 deletions server/manager/src/main/java/org/apache/accumulo/manager/upgrade/Upgrader12to13.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,21 +23,23 @@
import static org.apache.accumulo.core.metadata.schema.MetadataSchema.RESERVED_PREFIX;
import static org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.Upgrade12to13.COMPACT_COL;

import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;

import org.apache.accumulo.core.Constants;
import org.apache.accumulo.core.client.BatchWriter;
import org.apache.accumulo.core.client.MutationsRejectedException;
import org.apache.accumulo.core.client.TableNotFoundException;
import org.apache.accumulo.core.client.admin.TabletAvailability;
import org.apache.accumulo.core.clientImpl.Namespace;
import org.apache.accumulo.core.conf.Property;
import org.apache.accumulo.core.data.Key;
import org.apache.accumulo.core.data.Mutation;
import org.apache.accumulo.core.data.TableId;
import org.apache.accumulo.core.data.Value;
import org.apache.accumulo.core.fate.zookeeper.ZooUtil;
import org.apache.accumulo.core.manager.state.tables.TableState;
import org.apache.accumulo.core.metadata.AccumuloTable;
import org.apache.accumulo.core.metadata.schema.Ample.DataLevel;
import org.apache.accumulo.core.metadata.schema.Ample.TabletsMutator;
Expand All @@ -51,7 +53,9 @@
import org.apache.accumulo.core.security.Authorizations;
import org.apache.accumulo.server.ServerContext;
import org.apache.accumulo.server.conf.store.TablePropKey;
import org.apache.accumulo.server.tables.TableManager;
import org.apache.accumulo.server.init.FileSystemInitializer;
import org.apache.accumulo.server.init.InitialConfiguration;
import org.apache.accumulo.server.init.ZooKeeperInitializer;
import org.apache.accumulo.server.util.PropUtil;
import org.apache.zookeeper.KeeperException;
import org.apache.zookeeper.data.Stat;
Expand Down Expand Up @@ -120,13 +124,25 @@ private static void addCompactionsNode(ServerContext context) {
}

private void createFateTable(ServerContext context) {
ZooKeeperInitializer zkInit = new ZooKeeperInitializer();
zkInit.initFateTableState(context);

try {
TableManager.prepareNewTableState(context, AccumuloTable.FATE.tableId(),
Namespace.ACCUMULO.id(), AccumuloTable.FATE.tableName(), TableState.ONLINE,
ZooUtil.NodeExistsPolicy.SKIP);
} catch (KeeperException | InterruptedException e) {
throw new IllegalStateException("Error creating table: " + AccumuloTable.FATE.tableName(), e);
FileSystemInitializer initializer = new FileSystemInitializer(
new InitialConfiguration(context.getHadoopConf(), context.getSiteConfiguration()));
FileSystemInitializer.InitialTablet fateTableTableTablet =
initializer.createFateRefTablet(context);
// Add references to the Metadata Table
try (BatchWriter writer = context.createBatchWriter(AccumuloTable.METADATA.tableName())) {
writer.addMutation(fateTableTableTablet.createMutation());
} catch (MutationsRejectedException | TableNotFoundException e) {
LOG.error("Failed to write tablet refs to metadata table");
throw new RuntimeException(e);
}
} catch (IOException e) {
LOG.error("Problem attempting to create Fate table", e);
}
LOG.info("Created Fate table");
}

private void removeCompactColumnsFromRootTabletMetadata(ServerContext context) {
Expand Down