Trusted node sync

Trusted node sync, aka checkpoint sync, allows syncing tyhe chain from a
trusted node instead of relying on a full sync from genesis.

Features include:

* sync from any slot, including the latest finalized slot
* backfill blocks either from the REST api (default) or p2p
* resume backfilling either while running the node as normal or offline

Future improvements:

* top up blocks between head in database and some other node - this
makes for an efficient backup tool
* recreate historical state to enable historical queries

Assorted fixes:

* don't store backfill root in database - use summaries to find backfill
point
* query remote peers for status only once per sync step
* fix peer status check in backward sync mode
* fix several off-by-ones in backward sync
* use common forked block/state reader in REST API
* disable JSON state readed to avoid the risk of stack overflows

beacon_chain/beacon_chain_db.nim

@@ -137,9 +137,7 @@ type
       ## only recent contract state data (i.e. only recent `deposit_roots`).
     kHashToStateDiff # Obsolete
     kHashToStateOnlyMutableValidators
-    kBackfillBlock
-      ## Pointer to the earliest block that we have backfilled - if this is not
-      ## set, backfill == tail
+    kBackfillBlock # Obsolete, was in `unstable` for a while, but never released
 
   BeaconBlockSummary* = object
     ## Cache of beacon block summaries - during startup when we construct the
@@ -477,7 +475,7 @@ proc close*(db: BeaconchainDB) =
 
   db.db = nil
 
-func toBeaconBlockSummary(v: SomeSomeBeaconBlock): BeaconBlockSummary =
+func toBeaconBlockSummary*(v: SomeSomeBeaconBlock): BeaconBlockSummary =
   BeaconBlockSummary(
     slot: v.slot,
     parent_root: v.parent_root,
@@ -591,9 +589,6 @@ proc putTailBlock*(db: BeaconChainDB, key: Eth2Digest) =
 proc putGenesisBlock*(db: BeaconChainDB, key: Eth2Digest) =
   db.keyValues.putRaw(subkey(kGenesisBlock), key)
 
-proc putBackfillBlock*(db: BeaconChainDB, key: Eth2Digest) =
-  db.keyValues.putRaw(subkey(kBackfillBlock), key)
-
 proc putEth2FinalizedTo*(db: BeaconChainDB,
                          eth1Checkpoint: DepositContractSnapshot) =
   db.keyValues.putSnappySSZ(subkey(kDepositsFinalizedByEth2), eth1Checkpoint)
@@ -797,9 +792,6 @@ proc getGenesisBlock*(db: BeaconChainDB): Opt[Eth2Digest] =
   db.keyValues.getRaw(subkey(kGenesisBlock), Eth2Digest) or
     db.v0.getGenesisBlock()
 
-proc getBackfillBlock*(db: BeaconChainDB): Opt[Eth2Digest] =
-  db.keyValues.getRaw(subkey(kBackfillBlock), Eth2Digest)
-
 proc getEth2FinalizedTo(db: BeaconChainDBV0): Opt[DepositContractSnapshot] =
   result.ok(DepositContractSnapshot())
   let r = db.backend.getSnappySSZ(subkey(kDepositsFinalizedByEth2), result.get)
@@ -856,7 +848,7 @@ iterator getAncestors*(db: BeaconChainDB, root: Eth2Digest):
     yield res
     root = res.message.parent_root
 
-proc loadSummaries(db: BeaconChainDB): Table[Eth2Digest, BeaconBlockSummary] =
+proc loadSummaries*(db: BeaconChainDB): Table[Eth2Digest, BeaconBlockSummary] =
   # Load summaries into table - there's no telling what order they're in so we
   # load them all - bugs in nim prevent this code from living in the iterator.
   var summaries = initTable[Eth2Digest, BeaconBlockSummary](1024*1024)

beacon_chain/beacon_node.nim

@@ -60,6 +60,7 @@ type
     vcProcess*: Process
     requestManager*: RequestManager
     syncManager*: SyncManager[Peer, PeerID]
+    backfiller*: SyncManager[Peer, PeerID]
     genesisSnapshotContent*: string
     actionTracker*: ActionTracker
     processor*: ref Eth2Processor

beacon_chain/conf.nim

@@ -36,14 +36,15 @@ const
   defaultSigningNodeRequestTimeout* = 60
 
 type
-  BNStartUpCmd* = enum
+  BNStartUpCmd* {.pure.} = enum
     noCommand
     createTestnet
     deposits
     wallets
     record
     web3
     slashingdb
+    trustedNodeSync
 
   WalletsCmd* {.pure.} = enum
     create  = "Creates a new EIP-2386 wallet"
@@ -176,9 +177,9 @@ type
 
     case cmd* {.
       command
-      defaultValue: noCommand }: BNStartUpCmd
+      defaultValue: BNStartUpCmd.noCommand }: BNStartUpCmd
 
-    of noCommand:
+    of BNStartUpCmd.noCommand:
       bootstrapNodes* {.
         desc: "Specifies one or more bootstrap nodes to use when connecting to the network"
         abbr: "b"
@@ -231,12 +232,10 @@ type
         name: "weak-subjectivity-checkpoint" }: Option[Checkpoint]
 
       finalizedCheckpointState* {.
-        hidden # TODO unhide when backfilling is done
         desc: "SSZ file specifying a recent finalized state"
         name: "finalized-checkpoint-state" }: Option[InputFile]
 
       finalizedCheckpointBlock* {.
-        hidden # TODO unhide when backfilling is done
         desc: "SSZ file specifying a recent finalized block"
         name: "finalized-checkpoint-block" }: Option[InputFile]
 
@@ -372,7 +371,7 @@ type
         name: "terminal-total-difficulty-override"
       }: Option[uint64]
 
-    of createTestnet:
+    of BNStartUpCmd.createTestnet:
       testnetDepositsFile* {.
         desc: "A LaunchPad deposits file for the genesis state validators"
         name: "deposits-file" }: InputFile
@@ -406,7 +405,7 @@ type
         desc: "Output file with list of bootstrap nodes for the network"
         name: "output-bootstrap-file" }: OutFile
 
-    of wallets:
+    of BNStartUpCmd.wallets:
       case walletsCmd* {.command.}: WalletsCmd
       of WalletsCmd.create:
         nextAccount* {.
@@ -439,7 +438,7 @@ type
       of WalletsCmd.list:
         discard
 
-    of deposits:
+    of BNStartUpCmd.deposits:
       case depositsCmd* {.command.}: DepositsCmd
       of DepositsCmd.createTestnetDeposits:
         totalDeposits* {.
@@ -498,7 +497,7 @@ type
           name: "epoch"
           desc: "The desired exit epoch" }: Option[uint64]
 
-    of record:
+    of BNStartUpCmd.record:
       case recordCmd* {.command.}: RecordCmd
       of RecordCmd.create:
         ipExt* {.
@@ -528,15 +527,15 @@ type
           desc: "ENR URI of the record to print"
           name: "enr" .}: Record
 
-    of web3:
+    of BNStartUpCmd.web3:
       case web3Cmd* {.command.}: Web3Cmd
       of Web3Cmd.test:
         web3TestUrl* {.
           argument
           desc: "The web3 provider URL to test"
           name: "url" }: Uri
 
-    of slashingdb:
+    of BNStartUpCmd.slashingdb:
       case slashingdbCmd* {.command.}: SlashProtCmd
       of SlashProtCmd.`import`:
         importedInterchangeFile* {.
@@ -552,6 +551,23 @@ type
           desc: "EIP-3076 slashing protection interchange file to export"
           argument }: OutFile
 
+    of BNStartUpCmd.trustedNodeSync:
+      trustedNodeUrl* {.
+        desc: "URL of the REST API to sync from"
+        defaultValue: "http://localhost:5052"
+        name: "trusted-node-url"
+      .}: string
+
+      blockId* {.
+        desc: "Block id to sync to - this can be a block root, slot number, \"finalized\" or \"head\""
+        defaultValue: "finalized"
+      .}: string
+
+      backfillBlocks* {.
+        desc: "Backfill blocks directly from REST server instead of fetching via API"
+        defaultValue: true
+        name: "backfill"}: bool
+
   ValidatorClientConf* = object
     logLevel* {.
       desc: "Sets the log level"

beacon_chain/consensus_object_pools/block_clearance.nim

@@ -273,7 +273,7 @@ proc addBackfillBlock*(
   logScope:
     blockRoot = shortLog(signedBlock.root)
     blck = shortLog(signedBlock.message)
-    backfill = (dag.backfill.slot, shortLog(dag.backfill.root))
+    backfill = (dag.backfill.slot, shortLog(dag.backfill.parent_root))
 
   template blck(): untyped = signedBlock.message # shortcuts without copy
   template blockRoot(): untyped = signedBlock.root
@@ -292,7 +292,7 @@ proc addBackfillBlock*(
     debug "Block unviable or duplicate"
     return err(BlockError.UnviableFork)
 
-  if dag.backfill.root != signedBlock.root:
+  if dag.backfill.parent_root != signedBlock.root:
     debug "Block does not match expected backfill root"
     return err(BlockError.MissingParent) # MissingChild really, but ..
 
@@ -319,8 +319,7 @@ proc addBackfillBlock*(
     return err(BlockError.Invalid)
 
   dag.putBlock(signedBlock.asTrusted())
-  dag.db.putBackfillBlock(signedBlock.root)
-  dag.backfill = (blck.slot, blck.parent_root)
+  dag.backfill = blck.toBeaconBlockSummary()
 
   # Invariants maintained on startup
   doAssert dag.backfillBlocks.lenu64 == dag.tail.slot.uint64

beacon_chain/consensus_object_pools/block_pools_types.nim

@@ -108,14 +108,9 @@ type
     ## go - the tail block is unique in that its parent is set to `nil`, even
     ## in the case where a later genesis block exists.
 
-    backfill*: tuple[slot: Slot, root: Eth2Digest] ##\
-    ## The backfill is root of the parent of the the earliest block that we
-    ## have synced, when performing a checkpoint sync start. Because the
-    ## `tail` BlockRef does not have a parent, we store here the root of the
-    ## block we're expecting during backfill.
-    ## When starting a checkpoint sync, `backfill` == `tail.parent_root` - we
-    ## then sync backards, moving the backfill (but not tail!) until we hit
-    ## genesis at which point we set backfill to the zero hash.
+    backfill*: BeaconBlockSummary ##\
+    ## The backfill points to the oldest block that we have, in the database -
+    ## when backfilling, we'll be fetching its parent first
 
     heads*: seq[BlockRef] ##\
     ## Candidate heads of candidate chains

beacon_chain/consensus_object_pools/blockchain_dag.nim

@@ -353,7 +353,6 @@ proc init*(T: type ChainDAGRef, cfg: RuntimeConfig, db: BeaconChainDB,
   let
     tailBlockRoot = db.getTailBlock()
     headBlockRoot = db.getHeadBlock()
-    backfillBlockRoot = db.getBackfillBlock()
 
   doAssert tailBlockRoot.isSome(), "Missing tail block, database corrupt?"
   doAssert headBlockRoot.isSome(), "Missing head block, database corrupt?"
@@ -374,18 +373,6 @@ proc init*(T: type ChainDAGRef, cfg: RuntimeConfig, db: BeaconChainDB,
         "preInit should have initialized the database with a genesis block")
     withBlck(genesisBlock): BlockRef.init(genesisBlockRoot, blck.message)
 
-  let backfill =
-    if backfillBlockRoot.isSome():
-      let backfillBlock = db.getForkedBlock(backfillBlockRoot.get()).expect(
-        "backfill block must be present in database, database corrupt?")
-      (getForkedBlockField(backfillBlock, slot),
-        getForkedBlockField(backfillBlock, parentRoot))
-    elif tailRef.slot > GENESIS_SLOT:
-      (getForkedBlockField(tailBlock, slot),
-        getForkedBlockField(tailBlock, parentRoot))
-    else:
-      (GENESIS_SLOT, Eth2Digest())
-
   var
     blocks: HashSet[KeyedBlockRef]
     headRef: BlockRef
@@ -398,11 +385,15 @@ proc init*(T: type ChainDAGRef, cfg: RuntimeConfig, db: BeaconChainDB,
   var
     backfillBlocks = newSeq[Eth2Digest](tailRef.slot.int)
     curRef: BlockRef
+    backfill = BeaconBlockSummary(slot: GENESIS_SLOT)
 
   for blck in db.getAncestorSummaries(headRoot):
     if blck.summary.slot < tailRef.slot:
       backfillBlocks[blck.summary.slot.int] = blck.root
+      backfill = blck.summary
     elif blck.summary.slot == tailRef.slot:
+      backfill = blck.summary
+
       if curRef == nil:
         curRef = tailRef
         headRef = tailRef
@@ -562,7 +553,7 @@ proc init*(T: type ChainDAGRef, cfg: RuntimeConfig, db: BeaconChainDB,
     finalizedHead = shortLog(dag.finalizedHead),
     tail = shortLog(dag.tail),
     totalBlocks = dag.blocks.len(),
-    backfill = (dag.backfill.slot, shortLog(dag.backfill.root))
+    backfill = (dag.backfill.slot, shortLog(dag.backfill.parent_root))
 
   dag
 
@@ -1366,33 +1357,45 @@ proc updateHead*(
         dag.finalizedHead.slot.epoch)
       dag.onFinHappened(data)
 
-proc isInitialized*(T: type ChainDAGRef, db: BeaconChainDB): bool =
+proc isInitialized*(T: type ChainDAGRef, db: BeaconChainDB): Result[void, cstring] =
   # Lightweight check to see if we have the minimal information needed to
   # load up a database - we don't check head here - if something is wrong with
   # head, it's likely an initialized, but corrupt database - init will detect
   # that
   let
     genesisBlockRoot = db.getGenesisBlock()
-    tailBlockRoot = db.getTailBlock()
 
-  if not (genesisBlockRoot.isSome() and tailBlockRoot.isSome()):
-    return false
+  if not genesisBlockRoot.isSome():
+    return err("Genesis block root missing")
 
   let
     genesisBlock = db.getForkedBlock(genesisBlockRoot.get())
-    tailBlock = db.getForkedBlock(tailBlockRoot.get())
+  if not genesisBlock.isSome():
+    return err("Genesis block missing")
 
-  if not (genesisBlock.isSome() and tailBlock.isSome()):
-    return false
   let
     genesisStateRoot = withBlck(genesisBlock.get()): blck.message.state_root
+
+  if not db.containsState(genesisStateRoot):
+    return err("Genesis state missing")
+
+  let
+    tailBlockRoot = db.getTailBlock()
+  if not tailBlockRoot.isSome():
+    return err("Tail block root missing")
+
+  let
+    tailBlock = db.getForkedBlock(tailBlockRoot.get())
+  if not tailBlock.isSome():
+    return err("Tail block missing")
+
+  let
     tailStateRoot = withBlck(tailBlock.get()): blck.message.state_root
 
-  if not (
-      db.containsState(genesisStateRoot) and db.containsState(tailStateRoot)):
-    return false
+  if not db.containsState(tailStateRoot):
+    return err("Tail state missing")
 
-  true
+  ok()
 
 proc preInit*(
     T: type ChainDAGRef, db: BeaconChainDB,
@@ -1537,3 +1540,6 @@ proc aggregateAll*(
     err("aggregate: no attesting keys")
   else:
     ok(finish(aggregateKey))
+
+func needsBackfill*(dag: ChainDAGRef): bool =
+  dag.backfill.slot > dag.genesis.slot