(op-node) Keep consistent status when meet an unexpected el sync

op-node/rollup/derive/engine_controller.go

@@ -4,17 +4,19 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"strings"
 	"time"
 
+	"github.com/ethereum/go-ethereum"
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/log"
+
 	"github.com/ethereum-optimism/optimism/op-node/rollup"
 	"github.com/ethereum-optimism/optimism/op-node/rollup/async"
 	"github.com/ethereum-optimism/optimism/op-node/rollup/conductor"
 	"github.com/ethereum-optimism/optimism/op-node/rollup/sync"
 	"github.com/ethereum-optimism/optimism/op-service/clock"
 	"github.com/ethereum-optimism/optimism/op-service/eth"
-	"github.com/ethereum/go-ethereum"
-	"github.com/ethereum/go-ethereum/common"
-	"github.com/ethereum/go-ethereum/log"
 )
 
 type syncStatusEnum int
@@ -32,7 +34,14 @@ const (
 	syncStatusFinishedEL                // EL sync is done & we should be performing consolidation
 )
 
-var errNoFCUNeeded = errors.New("no FCU call was needed")
+var (
+	errNoFCUNeeded             = errors.New("no FCU call was needed")
+	ErrELSyncTriggerUnexpected = errors.New("forced head needed for startup")
+
+	maxFCURetryAttempts = 5
+	fcuRetryDelay       = 5 * time.Second
+	needSyncWithEngine  = false
+)
 
 var _ EngineControl = (*EngineController)(nil)
 var _ LocalEngineControl = (*EngineController)(nil)
@@ -279,6 +288,10 @@ func (e *EngineController) checkNewPayloadStatus(status eth.ExecutePayloadStatus
 		}
 		// Allow SYNCING and ACCEPTED if engine EL sync is enabled
 		return status == eth.ExecutionValid || status == eth.ExecutionSyncing || status == eth.ExecutionAccepted
+	} else if e.syncMode == sync.CLSync {
+		if status == eth.ExecutionInconsistent {
+			return true
+		}
 	}
 	return status == eth.ExecutionValid
 }
@@ -296,6 +309,16 @@ func (e *EngineController) checkForkchoiceUpdatedStatus(status eth.ExecutePayloa
 	return status == eth.ExecutionValid
 }
 
+// checkELSyncTriggered checks returned err of engine_newPayloadV1
+func (e *EngineController) checkELSyncTriggered(status eth.ExecutePayloadStatus, err error) bool {
+	if err == nil {
+		return false
+	} else if strings.Contains(err.Error(), ErrELSyncTriggerUnexpected.Error()) {
+		return e.syncMode != sync.ELSync && status == eth.ExecutionSyncing
+	}
+	return false
+}
+
 // checkUpdateUnsafeHead checks if we can update current unsafeHead for op-node
 func (e *EngineController) checkUpdateUnsafeHead(status eth.ExecutePayloadStatus) bool {
 	if e.syncMode == sync.ELSync {
@@ -361,11 +384,15 @@ func (e *EngineController) InsertUnsafePayload(ctx context.Context, envelope *et
 	// Insert the payload & then call FCU
 	status, err := e.engine.NewPayload(ctx, envelope.ExecutionPayload, envelope.ParentBeaconBlockRoot)
 	if err != nil {
-		return NewTemporaryError(fmt.Errorf("failed to update insert payload: %w", err))
+		if strings.Contains(err.Error(), ErrELSyncTriggerUnexpected.Error()) {
+			log.Info("el sync triggered as unexpected")
+		} else {
+			return NewTemporaryError(fmt.Errorf("failed to update insert payload: %w", err))
+		}
 	}
 
 	//process inconsistent state
-	if status.Status == eth.ExecutionInconsistent {
+	if status.Status == eth.ExecutionInconsistent || e.checkELSyncTriggered(status.Status, err) {
 		currentL2Info, err := e.getCurrentL2Info(ctx)
 		if err != nil {
 			return NewTemporaryError(fmt.Errorf("failed to process inconsistent state: %w", err))
@@ -388,13 +415,25 @@ func (e *EngineController) InsertUnsafePayload(ctx context.Context, envelope *et
 			FinalizedBlockHash: e.finalizedHead.Hash,
 		}
 
-		fcuRes, err := e.engine.ForkchoiceUpdate(ctx, &fcuReq, nil)
-		if fcuRes.PayloadStatus.Status == eth.ExecutionValid {
-			log.Info("engine processed data successfully")
-			e.needFCUCall = false
-			return nil
-		} else {
-			return NewTemporaryError(fmt.Errorf("engine failed to process inconsistent data: %w", err))
+		for attempts := 0; attempts < maxFCURetryAttempts; attempts++ {
+			fcuRes, err := e.engine.ForkchoiceUpdate(ctx, &fcuReq, nil)
+			if err != nil {
+				if strings.Contains(err.Error(), "context deadline exceeded") {
+					log.Warn("Failed to share forkchoice-updated signal, attempt %d: %v", attempts+1, err)
+					time.Sleep(fcuRetryDelay)
+					continue
+				}
+				return NewTemporaryError(fmt.Errorf("engine failed to process due to error: %w", err))
+			}
+
+			if fcuRes.PayloadStatus.Status == eth.ExecutionValid {
+				log.Info("engine processed data successfully")
+				e.needFCUCall = false
+				needSyncWithEngine = true
+				break
+			} else {
+				return NewTemporaryError(fmt.Errorf("engine failed to process inconsistent data"))
+			}
 		}
 	}
 
@@ -412,8 +451,8 @@ func (e *EngineController) InsertUnsafePayload(ctx context.Context, envelope *et
 	}
 
 	//update unsafe,safe,finalize and send fcu for sync
-	if status.Status == eth.ExecutionInconsistent {
-		log.Info("engine meet inconsistent here")
+	if needSyncWithEngine {
+		log.Info("engine meet inconsistent, sync status")
 		currentUnsafe, _ := e.engine.L2BlockRefByLabel(ctx, eth.Unsafe)
 		//reset unsafe
 		e.SetUnsafeHead(currentUnsafe)
@@ -424,6 +463,8 @@ func (e *EngineController) InsertUnsafePayload(ctx context.Context, envelope *et
 		fc.HeadBlockHash = currentUnsafe.Hash
 		fc.SafeBlockHash = currentUnsafe.Hash
 		fc.FinalizedBlockHash = currentUnsafe.Hash
+
+		needSyncWithEngine = false
 	}
 
 	if e.syncStatus == syncStatusFinishedELButNotFinalized {

op-service/sources/engine_client.go

@@ -3,6 +3,7 @@ package sources
 import (
 	"context"
 	"fmt"
+	"strings"
 	"time"
 
 	"github.com/ethereum/go-ethereum/common"
@@ -12,6 +13,7 @@ import (
 	"github.com/ethereum/go-ethereum/rpc"
 
 	"github.com/ethereum-optimism/optimism/op-node/rollup"
+	"github.com/ethereum-optimism/optimism/op-node/rollup/derive"
 	"github.com/ethereum-optimism/optimism/op-service/client"
 	"github.com/ethereum-optimism/optimism/op-service/eth"
 	"github.com/ethereum-optimism/optimism/op-service/sources/caching"
@@ -136,6 +138,9 @@ func (s *EngineAPIClient) NewPayload(ctx context.Context, payload *eth.Execution
 
 	e.Trace("Received payload execution result", "status", result.Status, "latestValidHash", result.LatestValidHash, "message", result.ValidationError)
 	if err != nil {
+		if strings.Contains(err.Error(), derive.ErrELSyncTriggerUnexpected.Error()) {
+			return &result, err
+		}
 		e.Error("Payload execution failed", "err", err)
 		return nil, fmt.Errorf("failed to execute payload: %w", err)
 	}