[DON'T MERGE] Migrating PeerDAS to Pectra + soft-forking

.circleci/config.yml

@@ -68,6 +68,16 @@ executors:
       architecture: "amd64"
       platform: "linux/amd64"
 
+  machine_large_executor_amd64:
+    machine:
+      image: ubuntu-2204:2024.01.1 # https://circleci.com/developer/machine/image/ubuntu-2204
+      docker_layer_caching: true
+    resource_class: large
+    working_directory: ~/project
+    environment:
+      architecture: "amd64"
+      platform: "linux/amd64"
+
   machine_executor_arm64:
     machine:
       image: ubuntu-2204:2024.01.1 # https://circleci.com/developer/machine/image/ubuntu-2204
@@ -237,13 +247,23 @@ jobs:
           path: build/test-results
 
   spotless:
-    executor: small_executor
+    executor: medium_executor
     steps:
       - prepare
       - run:
           name: Spotless
           command: |
-            ./gradlew --no-daemon --parallel checkMavenCoordinateCollisions spotlessCheck checkModuleDependencies
+            ./gradlew --no-daemon --parallel spotlessCheck
+      - notify
+
+  moduleChecks:
+    executor: small_executor
+    steps:
+      - prepare
+      - run:
+          name: Module Checks
+          command: |
+            ./gradlew --no-daemon --parallel checkMavenCoordinateCollisions checkModuleDependencies
       - notify
 
   dockerScan:
@@ -343,8 +363,8 @@ jobs:
           path: build/test-results
 
   acceptanceTests:
-    parallelism: 5
-    executor: machine_executor_amd64
+    parallelism: 4
+    executor: machine_large_executor_amd64
     steps:
       - install_java_21
       - prepare
@@ -562,12 +582,17 @@ workflows:
           filters:
             tags: &filters-release-tags
               only: /^[0-9]+\.[0-9]+\.[0-9]+(-[a-zA-Z0-9]+)?/
+      - moduleChecks:
+          filters:
+            tags:
+              <<: *filters-release-tags
       - spotless:
           filters:
             tags:
               <<: *filters-release-tags
       - windowsBuild:
           requires:
+            - moduleChecks
             - spotless
           filters:
             tags:
@@ -630,6 +655,7 @@ workflows:
             - acceptanceTests
             - referenceTests
             - extractAPISpec
+            - moduleChecks
             - spotless
             - windowsBuild
           context:
@@ -649,6 +675,7 @@ workflows:
             - acceptanceTests
             - referenceTests
             - extractAPISpec
+            - moduleChecks
             - spotless
             - windowsBuild
           context:
@@ -668,6 +695,7 @@ workflows:
             - acceptanceTests
             - referenceTests
             - extractAPISpec
+            - moduleChecks
             - spotless
             - windowsBuild
           context:
@@ -700,6 +728,7 @@ workflows:
             - acceptanceTests
             - referenceTests
             - extractAPISpec
+            - moduleChecks
             - spotless
             - windowsBuild
   nightly:

.codespell/.codespellrc

@@ -1,5 +1,5 @@
 [codespell]
-skip = .git,package-lock.json,LOG.old.*
+skip = .git,package-lock.json,LOG.old.*,venv
 count =
 quiet-level = 3
 ignore-words = ./.codespell/wordlist.txt

.github/workflows/codespell.yml

@@ -16,8 +16,13 @@ jobs:
       - name: Checkout the repository
         uses: actions/checkout@v4
 
-      - name: Install prerequisites
-        run: pip install codespell
+      - name: Set up a Python venv and install prerequisites
+        run: |
+          python3 -m venv venv
+          source venv/bin/activate
+          pip install codespell
 
       - name: Spell check
-        run: codespell --config=./.codespell/.codespellrc
+        run: |
+          source venv/bin/activate
+          codespell --config=./.codespell/.codespellrc

CHANGELOG.md

@@ -4,18 +4,12 @@
 
 ## Current Releases
 
-For information on changes in released versions of Teku, see
-the [releases page](https://github.com/Consensys/teku/releases).
-
 ## Unreleased Changes
 
 ### Breaking Changes
 
 ### Additions and Improvements
-- Increased the executor queue default maximum size to 40_000 (previously 20_000), and other queues to 10_000 (previously 5_000). If you have custom settings for these queues, check to ensure they're still required.
-- Added `peers_direction_current` libp2p metric to track the number of peers by direction (inbound and outbound).
-- Deposit tree snapshots will be loaded from database as a default unless custom snapshot has been provided.
-- Added hidden option `--Xdeposit-contract-logs-syncing-enabled` to allow disabling the syncing of the deposit contract logs from the EL. This is useful when running a non-validating node. It is advisable to be used alongside with `--Xeth1-missing-deposits-event-logging-enabled=false` to avoid unnecessary logging of missing deposits.
-- Updated the bootnodes for Chiado and Gnosis networks
+- improve block publishing performance, especially relevant with locally produced blocks
 
 ### Bug Fixes
+- Added a startup script for unix systems to ensure that when jemalloc is installed the script sets the LD_PRELOAD environment variable to the use the jemalloc library

acceptance-tests/build.gradle

@@ -19,6 +19,7 @@ dependencies {
   testFixturesImplementation project(':infrastructure:metrics')
   testFixturesImplementation testFixtures(project(':infrastructure:async'))
   testFixturesImplementation project(':teku')
+  testFixturesImplementation project(':ethereum:json-types')
 
   testFixturesImplementation project(':infrastructure:crypto')
   testFixturesImplementation project(':infrastructure:time')
@@ -27,7 +28,7 @@ dependencies {
   testFixturesImplementation 'io.libp2p:jvm-libp2p'
   testFixturesImplementation 'org.apache.commons:commons-lang3'
   testFixturesImplementation 'commons-io:commons-io'
-  testFixturesImplementation 'org.apache.tuweni:tuweni-bytes'
+  testFixturesImplementation 'io.tmio:tuweni-bytes'
   testFixturesImplementation 'org.junit.jupiter:junit-jupiter-api'
   testFixturesImplementation 'org.testcontainers:testcontainers'
   testFixturesImplementation 'org.testcontainers:junit-jupiter'

acceptance-tests/src/acceptance-test/java/tech/pegasys/teku/test/acceptance/BlockProposalAcceptanceTest.java

@@ -21,8 +21,9 @@
 import java.util.Arrays;
 import java.util.Locale;
 import org.apache.tuweni.bytes.Bytes32;
-import org.junit.jupiter.api.Test;
-import tech.pegasys.teku.api.schema.eip7594.SignedBeaconBlockEip7594;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.ValueSource;
+import tech.pegasys.teku.infrastructure.bytes.Bytes20;
 import tech.pegasys.teku.infrastructure.unsigned.UInt64;
 import tech.pegasys.teku.test.acceptance.dsl.AcceptanceTestBase;
 import tech.pegasys.teku.test.acceptance.dsl.GenesisGenerator.InitialStateData;
@@ -34,8 +35,9 @@
 public class BlockProposalAcceptanceTest extends AcceptanceTestBase {
   private static final URL JWT_FILE = Resources.getResource("auth/ee-jwt-secret.hex");
 
-  @Test
-  void shouldHaveCorrectFeeRecipientAndGraffiti() throws Exception {
+  @ParameterizedTest(name = "ssz_encode={0}")
+  @ValueSource(booleans = {true, false})
+  void shouldHaveCorrectFeeRecipientAndGraffiti(final boolean useSszBlocks) throws Exception {
     final String networkName = "swift";
 
     final ValidatorKeystores validatorKeystores =
@@ -45,10 +47,7 @@ void shouldHaveCorrectFeeRecipientAndGraffiti() throws Exception {
         createGenesisGenerator()
             .network(networkName)
             .withAltairEpoch(UInt64.ZERO)
-            .withBellatrixEpoch(UInt64.ONE)
-            .withCapellaEpoch(UInt64.valueOf(2))
-            .withDenebEpoch(UInt64.valueOf(3))
-            .withEip7594Epoch(UInt64.valueOf(4))
+            .withBellatrixEpoch(UInt64.ZERO)
             .validatorKeys(validatorKeystores, validatorKeystores)
             .generate();
 
@@ -61,14 +60,8 @@ void shouldHaveCorrectFeeRecipientAndGraffiti() throws Exception {
                 .withJwtSecretFile(JWT_FILE)
                 .withNetwork(networkName)
                 .withInitialState(genesis)
-                .withRealNetwork()
                 .withAltairEpoch(UInt64.ZERO)
-                .withBellatrixEpoch(UInt64.ONE)
-                .withCapellaEpoch(UInt64.valueOf(2))
-                .withDenebEpoch(UInt64.valueOf(3))
-                .withEip7594Epoch(UInt64.valueOf(4))
-                .withTotalTerminalDifficulty(0)
-                .withTrustedSetupFromClasspath("mainnet-trusted-setup.txt")
+                .withBellatrixEpoch(UInt64.ZERO)
                 .withValidatorProposerDefaultFeeRecipient(defaultFeeRecipient)
                 .build());
     final TekuValidatorNode validatorClient =
@@ -78,22 +71,26 @@ void shouldHaveCorrectFeeRecipientAndGraffiti() throws Exception {
                 .withValidatorProposerDefaultFeeRecipient(defaultFeeRecipient)
                 .withInteropModeDisabled()
                 .withBeaconNodes(beaconNode)
+                .withBeaconNodeSszBlocksEnabled(useSszBlocks)
                 .withGraffiti(userGraffiti)
                 .withNetwork("auto")
                 .build());
 
     beaconNode.start();
     validatorClient.start();
 
-    beaconNode.waitForEpochAtOrAbove(4);
     beaconNode.waitForBlockSatisfying(
         block -> {
-          assertThat(block).isInstanceOf(SignedBeaconBlockEip7594.class);
-          final SignedBeaconBlockEip7594 eip7594Block = (SignedBeaconBlockEip7594) block;
-          assertThat(
-                  eip7594Block.getMessage().getBody().executionPayload.feeRecipient.toHexString())
+          final Bytes20 feeRecipient =
+              block
+                  .getMessage()
+                  .getBody()
+                  .getOptionalExecutionPayload()
+                  .orElseThrow()
+                  .getFeeRecipient();
+          assertThat(feeRecipient.toHexString().toLowerCase(Locale.ROOT))
               .isEqualTo(defaultFeeRecipient.toLowerCase(Locale.ROOT));
-          final Bytes32 graffiti = eip7594Block.getMessage().getBody().graffiti;
+          final Bytes32 graffiti = block.getMessage().getBody().getGraffiti();
           final String graffitiMessage =
               new String(
                   Arrays.copyOfRange(

acceptance-tests/src/acceptance-test/java/tech/pegasys/teku/test/acceptance/CapellaRemoteSignerAcceptanceTest.java

@@ -0,0 +1,132 @@
+/*
+ * Copyright Consensys Software Inc., 2022
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
+ * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations under the License.
+ */
+
+package tech.pegasys.teku.test.acceptance;
+
+import com.google.common.io.Resources;
+import java.net.URL;
+import java.util.Map;
+import org.junit.jupiter.api.Test;
+import tech.pegasys.teku.ethereum.execution.types.Eth1Address;
+import tech.pegasys.teku.infrastructure.time.SystemTimeProvider;
+import tech.pegasys.teku.infrastructure.unsigned.UInt64;
+import tech.pegasys.teku.test.acceptance.dsl.AcceptanceTestBase;
+import tech.pegasys.teku.test.acceptance.dsl.BesuDockerVersion;
+import tech.pegasys.teku.test.acceptance.dsl.BesuNode;
+import tech.pegasys.teku.test.acceptance.dsl.GenesisGenerator.InitialStateData;
+import tech.pegasys.teku.test.acceptance.dsl.TekuBeaconNode;
+import tech.pegasys.teku.test.acceptance.dsl.TekuNodeConfig;
+import tech.pegasys.teku.test.acceptance.dsl.TekuNodeConfigBuilder;
+import tech.pegasys.teku.test.acceptance.dsl.Web3SignerNode;
+import tech.pegasys.teku.test.acceptance.dsl.tools.ValidatorKeysApi;
+import tech.pegasys.teku.test.acceptance.dsl.tools.deposits.ValidatorKeystores;
+
+public class CapellaRemoteSignerAcceptanceTest extends AcceptanceTestBase {
+
+  private static final String NETWORK_NAME = "swift";
+  public static final Eth1Address WITHDRAWAL_ADDRESS =
+      Eth1Address.fromHexString("0xeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee");
+  private static final URL JWT_FILE = Resources.getResource("auth/ee-jwt-secret.hex");
+
+  @Test
+  void capellaWithRemoteSigner() throws Exception {
+    final UInt64 currentTime = new SystemTimeProvider().getTimeInSeconds();
+    final int genesisTime =
+        currentTime.intValue() + 30; // genesis needs added time for nodes to startup
+
+    final Web3SignerNode web3SignerNode =
+        createWeb3SignerNode(
+            config ->
+                config
+                    .withNetwork(NETWORK_NAME)
+                    .withAltairEpoch(UInt64.ZERO)
+                    .withBellatrixEpoch(UInt64.ZERO)
+                    .withCapellaEpoch(UInt64.ZERO));
+
+    web3SignerNode.start();
+    final ValidatorKeysApi signerApi = web3SignerNode.getValidatorKeysApi();
+    final BesuNode besuNode = createBesuNode(genesisTime);
+    besuNode.start();
+
+    final ValidatorKeystores validatorKeys =
+        createTekuDepositSender(NETWORK_NAME).generateValidatorKeys(4, WITHDRAWAL_ADDRESS);
+
+    signerApi.addLocalValidatorsAndExpect(validatorKeys, "imported");
+    signerApi.assertLocalValidatorListing(validatorKeys.getPublicKeys());
+
+    final InitialStateData initialStateData =
+        createGenesisGenerator()
+            .network(NETWORK_NAME)
+            .withGenesisTime(genesisTime)
+            .genesisDelaySeconds(0)
+            .withAltairEpoch(UInt64.ZERO)
+            .withBellatrixEpoch(UInt64.ZERO)
+            .withCapellaEpoch(UInt64.ZERO)
+            .withTotalTerminalDifficulty(0)
+            .genesisExecutionPayloadHeaderSource(besuNode::createGenesisExecutionPayload)
+            .validatorKeys(validatorKeys)
+            .generate();
+
+    final TekuBeaconNode tekuNode =
+        createTekuBeaconNode(
+            beaconNode(
+                genesisTime, besuNode, initialStateData, web3SignerNode.getValidatorRestApiUrl()));
+
+    tekuNode.start();
+
+    tekuNode.waitForNextEpoch();
+    tekuNode.waitForNewBlock();
+    tekuNode.waitForFullSyncCommitteeAggregate();
+  }
+
+  private BesuNode createBesuNode(final int genesisTime) {
+    final int shanghai =
+        genesisTime + 2; // 4 slots, 2 seconds each (swift) - activate Prague on first slot
+    final Map<String, String> genesisOverrides = Map.of("shanghaiTime", String.valueOf(shanghai));
+
+    return createBesuNode(
+        BesuDockerVersion.STABLE,
+        config ->
+            config
+                .withMergeSupport()
+                .withGenesisFile("besu/mergedGenesis.json")
+                .withP2pEnabled(true)
+                .withJwtTokenAuthorization(JWT_FILE),
+        genesisOverrides);
+  }
+
+  private static TekuNodeConfig beaconNode(
+      final int genesisTime,
+      final BesuNode besuNode,
+      final InitialStateData initialStateData,
+      final String signerUrl)
+      throws Exception {
+    return TekuNodeConfigBuilder.createBeaconNode()
+        .withInitialState(initialStateData)
+        .withInteropModeDisabled()
+        .withNetwork(NETWORK_NAME)
+        .withAltairEpoch(UInt64.ZERO)
+        .withBellatrixEpoch(UInt64.ZERO)
+        .withCapellaEpoch(UInt64.ZERO)
+        .withTotalTerminalDifficulty(0)
+        .withGenesisTime(genesisTime)
+        .withExecutionEngine(besuNode)
+        .withJwtSecretFile(JWT_FILE)
+        .withExternalSignerUrl(signerUrl)
+        .withExternalSignerPublicKeys("external-signer")
+        .withValidatorProposerDefaultFeeRecipient("0xFE3B557E8Fb62b89F4916B721be55cEb828dBd73")
+        .withStartupTargetPeerCount(0)
+        .withRealNetwork()
+        .build();
+  }
+}