Feat: Shadow block recovery mechanism

.github/actions/dockerfiles/Dockerfile.debian-source

@@ -24,5 +24,5 @@ RUN --mount=type=tmpfs,target=${BUILD_DIR} cp -R /src/. ${BUILD_DIR}/ \
     && cp -R ${BUILD_DIR}/target/${TARGET}/release/. /out
 
 FROM --platform=${TARGETPLATFORM} debian:bookworm
-COPY --from=build /out/stacks-node /out/stacks-signer /bin/
+COPY --from=build /out/stacks-node /out/stacks-signer /out/stacks-inspect /bin/
 CMD ["stacks-node", "mainnet"]

.github/workflows/bitcoin-tests.yml

@@ -139,6 +139,7 @@ jobs:
           - tests::nakamoto_integrations::utxo_check_on_startup_panic
           - tests::nakamoto_integrations::utxo_check_on_startup_recover
           - tests::nakamoto_integrations::v3_signer_api_endpoint
+          - tests::nakamoto_integrations::test_shadow_recovery
           - tests::nakamoto_integrations::signer_chainstate
           - tests::nakamoto_integrations::clarity_cost_spend_down
           - tests::nakamoto_integrations::v3_blockbyheight_api_endpoint

stacks-common/src/types/mod.rs

@@ -170,6 +170,21 @@ impl StacksEpochId {
         }
     }
 
+    /// Whether or not this epoch supports shadow blocks
+    pub fn supports_shadow_blocks(&self) -> bool {
+        match self {
+            StacksEpochId::Epoch10
+            | StacksEpochId::Epoch20
+            | StacksEpochId::Epoch2_05
+            | StacksEpochId::Epoch21
+            | StacksEpochId::Epoch22
+            | StacksEpochId::Epoch23
+            | StacksEpochId::Epoch24
+            | StacksEpochId::Epoch25 => false,
+            StacksEpochId::Epoch30 => true,
+        }
+    }
+
     /// Does this epoch support unlocking PoX contributors that miss a slot?
     ///
     /// Epoch 2.0 - 2.05 didn't support this feature, but they weren't epoch-guarded on it. Instead,

stackslib/src/burnchains/mod.rs

@@ -450,6 +450,7 @@ impl PoxConstants {
         )
     }
 
+    // NOTE: this is the *old* pre-Nakamoto testnet
     pub fn testnet_default() -> PoxConstants {
         PoxConstants::new(
             POX_REWARD_CYCLE_LENGTH / 2,   // 1050
@@ -468,6 +469,10 @@ impl PoxConstants {
         ) // total liquid supply is 40000000000000000 µSTX
     }
 
+    pub fn nakamoto_testnet_default() -> PoxConstants {
+        PoxConstants::new(900, 100, 51, 100, 0, u64::MAX, u64::MAX, 242, 243, 246, 244)
+    }
+
     // TODO: add tests from mutation testing results #4838
     #[cfg_attr(test, mutants::skip)]
     pub fn regtest_default() -> PoxConstants {

stackslib/src/burnchains/tests/mod.rs

@@ -351,10 +351,30 @@ impl TestMinerFactory {
 
 impl TestBurnchainBlock {
     pub fn new(parent_snapshot: &BlockSnapshot, fork_id: u64) -> TestBurnchainBlock {
+        let burn_header_hash = BurnchainHeaderHash::from_test_data(
+            parent_snapshot.block_height + 1,
+            &parent_snapshot.index_root,
+            fork_id,
+        );
         TestBurnchainBlock {
             parent_snapshot: parent_snapshot.clone(),
             block_height: parent_snapshot.block_height + 1,
-            txs: vec![],
+            txs: vec![
+                // make sure that no block-commit gets vtxindex == 0 unless explicitly structured.
+                // This prestx mocks a burnchain coinbase
+                BlockstackOperationType::PreStx(PreStxOp {
+                    output: StacksAddress::burn_address(false),
+                    txid: Txid::from_test_data(
+                        parent_snapshot.block_height + 1,
+                        0,
+                        &burn_header_hash,
+                        128,
+                    ),
+                    vtxindex: 0,
+                    block_height: parent_snapshot.block_height + 1,
+                    burn_header_hash,
+                }),
+            ],
             fork_id: fork_id,
             timestamp: get_epoch_time_secs(),
         }
@@ -397,6 +417,7 @@ impl TestBurnchainBlock {
         parent_block_snapshot: Option<&BlockSnapshot>,
         new_seed: Option<VRFSeed>,
         epoch_marker: u8,
+        parent_is_shadow: bool,
     ) -> LeaderBlockCommitOp {
         let pubks = miner
             .privks
@@ -435,6 +456,13 @@ impl TestBurnchainBlock {
         )
         .expect("FATAL: failed to read block commit");
 
+        if parent_is_shadow {
+            assert!(
+                get_commit_res.is_none(),
+                "FATAL: shadow parent should not have a block-commit"
+            );
+        }
+
         let input = SortitionDB::get_last_block_commit_by_sender(ic.conn(), &apparent_sender)
             .unwrap()
             .map(|commit| (commit.txid.clone(), 1 + (commit.commit_outs.len() as u32)))
@@ -454,7 +482,8 @@ impl TestBurnchainBlock {
                     block_hash,
                     self.block_height,
                     &new_seed,
-                    &parent,
+                    parent.block_height as u32,
+                    parent.vtxindex as u16,
                     leader_key.block_height as u32,
                     leader_key.vtxindex as u16,
                     burn_fee,
@@ -464,16 +493,42 @@ impl TestBurnchainBlock {
                 txop
             }
             None => {
-                // initial
-                let txop = LeaderBlockCommitOp::initial(
-                    block_hash,
-                    self.block_height,
-                    &new_seed,
-                    leader_key,
-                    burn_fee,
-                    &input,
-                    &apparent_sender,
-                );
+                let txop = if parent_is_shadow {
+                    test_debug!(
+                        "Block-commit for {} (burn height {}) builds on shadow sortition",
+                        block_hash,
+                        self.block_height
+                    );
+
+                    LeaderBlockCommitOp::new(
+                        block_hash,
+                        self.block_height,
+                        &new_seed,
+                        last_snapshot_with_sortition.block_height as u32,
+                        0,
+                        leader_key.block_height as u32,
+                        leader_key.vtxindex as u16,
+                        burn_fee,
+                        &input,
+                        &apparent_sender,
+                    )
+                } else {
+                    // initial
+                    test_debug!(
+                        "Block-commit for {} (burn height {}) builds on genesis",
+                        block_hash,
+                        self.block_height,
+                    );
+                    LeaderBlockCommitOp::initial(
+                        block_hash,
+                        self.block_height,
+                        &new_seed,
+                        leader_key,
+                        burn_fee,
+                        &input,
+                        &apparent_sender,
+                    )
+                };
                 txop
             }
         };
@@ -517,6 +572,7 @@ impl TestBurnchainBlock {
             parent_block_snapshot,
             None,
             STACKS_EPOCH_2_4_MARKER,
+            false,
         )
     }
 

stackslib/src/chainstate/burn/operations/leader_block_commit.rs

@@ -136,7 +136,8 @@ impl LeaderBlockCommitOp {
         block_header_hash: &BlockHeaderHash,
         block_height: u64,
         new_seed: &VRFSeed,
-        parent: &LeaderBlockCommitOp,
+        parent_block_height: u32,
+        parent_vtxindex: u16,
         key_block_ptr: u32,
         key_vtxindex: u16,
         burn_fee: u64,
@@ -148,8 +149,8 @@ impl LeaderBlockCommitOp {
             new_seed: new_seed.clone(),
             key_block_ptr: key_block_ptr,
             key_vtxindex: key_vtxindex,
-            parent_block_ptr: parent.block_height as u32,
-            parent_vtxindex: parent.vtxindex as u16,
+            parent_block_ptr: parent_block_height,
+            parent_vtxindex: parent_vtxindex,
             memo: vec![],
             burn_fee: burn_fee,
             input: input.clone(),
@@ -696,8 +697,19 @@ impl LeaderBlockCommitOp {
         //  is descendant
         let directly_descended_from_anchor = epoch_id.block_commits_to_parent()
             && self.block_header_hash == reward_set_info.anchor_block;
-        let descended_from_anchor = directly_descended_from_anchor || tx
-            .descended_from(parent_block_height, &reward_set_info.anchor_block)
+
+        // second, if we're in a nakamoto epoch, and the parent block has vtxindex 0 (i.e. the
+        // coinbase of the burnchain block), then assume that this block descends from the anchor
+        // block for the purposes of validating its PoX payouts.  The block validation logic will
+        // check that the parent block is indeed a shadow block, and that `self.parent_block_ptr`
+        // points to the shadow block's tenure's burnchain block.
+        let maybe_shadow_parent = epoch_id.supports_shadow_blocks()
+            && self.parent_block_ptr != 0
+            && self.parent_vtxindex == 0;
+
+        let descended_from_anchor = directly_descended_from_anchor
+            || maybe_shadow_parent
+            || tx.descended_from(parent_block_height, &reward_set_info.anchor_block)
             .map_err(|e| {
                 error!("Failed to check whether parent (height={}) is descendent of anchor block={}: {}",
                        parent_block_height, &reward_set_info.anchor_block, e);
@@ -1031,10 +1043,12 @@ impl LeaderBlockCommitOp {
             return Err(op_error::BlockCommitNoParent);
         } else if self.parent_block_ptr != 0 || self.parent_vtxindex != 0 {
             // not building off of genesis, so the parent block must exist
+            // unless the parent is a shadow block
             let has_parent = tx
                 .get_block_commit_parent(parent_block_height, self.parent_vtxindex.into(), &tx_tip)?
                 .is_some();
-            if !has_parent {
+            let maybe_shadow_block = self.parent_vtxindex == 0 && epoch_id.supports_shadow_blocks();
+            if !has_parent && !maybe_shadow_block {
                 warn!("Invalid block commit: no parent block in this fork";
                       "apparent_sender" => %apparent_sender_repr
                 );

stackslib/src/chainstate/nakamoto/coordinator/mod.rs

@@ -58,6 +58,9 @@ use crate::monitoring::increment_stx_blocks_processed_counter;
 use crate::net::Error as NetError;
 use crate::util_lib::db::Error as DBError;
 
+#[cfg(any(test, feature = "testing"))]
+pub static TEST_COORDINATOR_STALL: std::sync::Mutex<Option<bool>> = std::sync::Mutex::new(None);
+
 #[cfg(test)]
 pub mod tests;
 
@@ -484,7 +487,14 @@ pub fn load_nakamoto_reward_set<U: RewardSetProvider>(
     let Some(anchor_block_header) = prepare_phase_sortitions
         .into_iter()
         .find_map(|sn| {
-            if !sn.sortition {
+            let shadow_tenure = match chain_state.nakamoto_blocks_db().is_shadow_tenure(&sn.consensus_hash) {
+                Ok(x) => x,
+                Err(e) => {
+                    return Some(Err(e));
+                }
+            };
+
+            if !sn.sortition && !shadow_tenure {
                 return None
             }
 
@@ -757,6 +767,21 @@ impl<
         true
     }
 
+    #[cfg(any(test, feature = "testing"))]
+    fn fault_injection_pause_nakamoto_block_processing() {
+        if *TEST_COORDINATOR_STALL.lock().unwrap() == Some(true) {
+            // Do an extra check just so we don't log EVERY time.
+            warn!("Coordinator is stalled due to testing directive");
+            while *TEST_COORDINATOR_STALL.lock().unwrap() == Some(true) {
+                std::thread::sleep(std::time::Duration::from_millis(10));
+            }
+            warn!("Coordinator is no longer stalled due to testing directive. Continuing...");
+        }
+    }
+
+    #[cfg(not(any(test, feature = "testing")))]
+    fn fault_injection_pause_nakamoto_block_processing() {}
+
     /// Handle one or more new Nakamoto Stacks blocks.
     /// If we process a PoX anchor block, then return its block hash.  This unblocks processing the
     /// next reward cycle's burnchain blocks.  Subsequent calls to this function will terminate
@@ -769,6 +794,8 @@ impl<
         );
 
         loop {
+            Self::fault_injection_pause_nakamoto_block_processing();
+
             // process at most one block per loop pass
             let mut processed_block_receipt = match NakamotoChainState::process_next_nakamoto_block(
                 &mut self.chain_state_db,