From dc302d3e04b89ff617e2cd3dfddb5eb5c03b2cde Mon Sep 17 00:00:00 2001 From: hehechen Date: Wed, 1 Mar 2023 15:14:28 +0800 Subject: [PATCH 01/12] compact log Signed-off-by: hehechen --- .../raftstore/src/coprocessor/dispatcher.rs | 16 ++++ components/raftstore/src/coprocessor/mod.rs | 8 ++ components/raftstore/src/store/fsm/apply.rs | 92 ++++++++++++++++++- components/raftstore/src/store/fsm/peer.rs | 5 +- .../src/core/forward_raft/region.rs | 5 + engine_store_ffi/src/ffi/mod.rs | 2 +- engine_store_ffi/src/observer.rs | 6 ++ proxy_ffi/src/apply_router_helper.rs | 51 ++++++++++ proxy_ffi/src/engine_store_helper_impls.rs | 8 ++ proxy_ffi/src/interfaces.rs | 21 +++++ proxy_ffi/src/lib.rs | 1 + proxy_ffi/src/raftstore_proxy.rs | 9 ++ proxy_ffi/src/raftstore_proxy_helper_impls.rs | 31 +++++++ proxy_server/src/run.rs | 3 + .../ffi/src/RaftStoreProxyFFI/ProxyFFI.h | 7 ++ 15 files changed, 261 insertions(+), 4 deletions(-) create mode 100644 proxy_ffi/src/apply_router_helper.rs diff --git a/components/raftstore/src/coprocessor/dispatcher.rs b/components/raftstore/src/coprocessor/dispatcher.rs index 0e45ef1d09d..1fc02443aa5 100644 --- a/components/raftstore/src/coprocessor/dispatcher.rs +++ b/components/raftstore/src/coprocessor/dispatcher.rs @@ -846,6 +846,22 @@ impl CoprocessorHost { } } + pub fn compact_log_in_queue(&self) -> bool { + for observer in &self.registry.message_observers { + let observer = observer.observer.inner(); + return observer.compact_log_in_queue(); + } + return true; + } + + pub fn get_compact_index_and_term(&self, region_id: u64, compact_index: u64, compact_term: u64) -> (u64, u64) { + for observer in &self.registry.message_observers { + let observer = observer.observer.inner(); + return observer.get_compact_index_and_term(region_id, compact_index, compact_term); + } + return (compact_index, compact_term); + } + pub fn shutdown(&self) { for entry in &self.registry.admin_observers { entry.observer.inner().stop(); diff --git a/components/raftstore/src/coprocessor/mod.rs b/components/raftstore/src/coprocessor/mod.rs index 98b045dbed8..d2ccbbe9d7b 100644 --- a/components/raftstore/src/coprocessor/mod.rs +++ b/components/raftstore/src/coprocessor/mod.rs @@ -343,6 +343,14 @@ pub trait MessageObserver: Coprocessor { fn on_raft_message(&self, _: &RaftMessage) -> bool { true } + + /// Returns true if need to get compact log index from pending queue. + fn compact_log_in_queue(&self) -> bool { + true + } + fn get_compact_index_and_term(&self, region_id: u64, compact_index: u64, compact_term: u64) -> (u64, u64) { + return (compact_index, compact_term); + } } #[derive(Clone, Debug, Default)] diff --git a/components/raftstore/src/store/fsm/apply.rs b/components/raftstore/src/store/fsm/apply.rs index 7afb188a4b0..1c330871fa5 100644 --- a/components/raftstore/src/store/fsm/apply.rs +++ b/components/raftstore/src/store/fsm/apply.rs @@ -996,6 +996,9 @@ where /// The counter of pending request snapshots. See more in `Peer`. pending_request_snapshot_count: Arc, + /// The max compact index from leader + max_compact_index: u64, + max_compact_term: u64, /// Indicates the peer is in merging, if that compact log won't be /// performed. is_merging: bool, @@ -1063,6 +1066,8 @@ where ready_source_region_id: 0, yield_state: None, wait_merge_state: None, + max_compact_index: 0, + max_compact_term: 0, is_merging: reg.is_merging, pending_cmds: PendingCmdQueue::new(), metrics: Default::default(), @@ -1420,6 +1425,21 @@ where let uuid = req.get_header().get_uuid().to_vec(); resp.mut_header().set_uuid(uuid); } + if req.has_admin_request() + { + let request = req.get_admin_request(); + let cmd_type = request.get_cmd_type(); + if cmd_type == AdminCmdType::CompactLog + { + let compact_index = request.get_compact_log().get_compact_index(); + let compact_term = request.get_compact_log().get_compact_term(); + if compact_index > self.max_compact_index + { + self.max_compact_index = compact_index; + self.max_compact_term = compact_term; + } + } + } (resp, ApplyResult::None) } else { ctx.exec_log_index = index; @@ -1685,7 +1705,7 @@ where AdminCmdType::ChangePeerV2 => self.exec_change_peer_v2(ctx, request), AdminCmdType::Split => self.exec_split(ctx, request), AdminCmdType::BatchSplit => self.exec_batch_split(ctx, request), - AdminCmdType::CompactLog => self.exec_compact_log(request), + AdminCmdType::CompactLog => self.exec_compact_log(ctx, request), AdminCmdType::TransferLeader => self.exec_transfer_leader(request, ctx.exec_log_term), AdminCmdType::ComputeHash => self.exec_compute_hash(ctx, request), AdminCmdType::VerifyHash => self.exec_verify_hash(ctx, request), @@ -3006,6 +3026,7 @@ where &mut self, voter_replicated_index: u64, voter_replicated_term: u64, + use_queue: bool, ) -> Result<(bool, Option>)> { PEER_ADMIN_CMD_COUNTER.compact.all.inc(); let first_index = entry_storage::first_index(&self.apply_state); @@ -3020,6 +3041,39 @@ where return Ok((false, None)); } + if !use_queue { + let mut compact_index = voter_replicated_index; + let mut compact_term = voter_replicated_term; + if compact_index > self.max_compact_index + { + compact_index = self.max_compact_index; + compact_term = self.max_compact_term; + } + if compact_index < first_index + { + debug!( + "compact_index < first index, no need to compact"; + "region_id" => self.region_id(), + "peer_id" => self.id(), + "compact_index" => compact_index, + "first_index" => first_index, + ); + return Ok((false, Some(ExecResult::HasPendingCompactCmd(false)))); + } + compact_raft_log( + &self.tag, + &mut self.apply_state, + compact_index, + compact_term, + )?; + PEER_ADMIN_CMD_COUNTER.compact.success.inc(); + return Ok((true, Some(ExecResult::CompactLog { + state: self.apply_state.get_truncated_state().clone(), + first_index, + has_pending: false, + }),)); + } + // When the witness restarted, the pending compact cmd has been lost, so use // `voter_replicated_index` for gc to avoid log accumulation. if !self.pending_cmds.has_compact() { @@ -3072,6 +3126,7 @@ where fn exec_compact_log( &mut self, + ctx: &mut ApplyContext, req: &AdminRequest, ) -> Result<(AdminResponse, ApplyResult)> { PEER_ADMIN_CMD_COUNTER.compact.all.inc(); @@ -3152,6 +3207,23 @@ where } } } + (compact_index, compact_term) = ctx.host.get_compact_index_and_term(self.region_id(), compact_index, compact_term); + if req.get_compact_log().get_compact_index() > compact_index && compact_index > self.max_compact_index + { + self.max_compact_index = req.get_compact_log().get_compact_index(); + self.max_compact_term = req.get_compact_log().get_compact_term(); + } + if compact_index < first_index { + debug!( + "compact index < first index, no need to compact"; + "region_id" => self.region_id(), + "peer_id" => self.id(), + "compact_index" => compact_index, + "first_index" => first_index, + "leader compact index" => req.get_compact_log().get_compact_index(), + ); + return Ok((resp, ApplyResult::None)); + } // compact failure is safe to be omitted, no need to assert. compact_raft_log( &self.tag, @@ -3717,6 +3789,7 @@ where region_id: u64, voter_replicated_index: u64, voter_replicated_term: u64, + applied_index: u64, }, } @@ -3790,6 +3863,7 @@ where region_id, voter_replicated_index, voter_replicated_term, + applied_index, } => { write!( f, @@ -4256,6 +4330,7 @@ where ctx: &mut ApplyContext, voter_replicated_index: u64, voter_replicated_term: u64, + applied_index: u64, ) { if self.delegate.pending_remove || self.delegate.stopped { return; @@ -4263,13 +4338,16 @@ where let res = self .delegate - .try_compact_log(voter_replicated_index, voter_replicated_term); + .try_compact_log(voter_replicated_index, voter_replicated_term, ctx.host.compact_log_in_queue()); match res { Ok((should_write, res)) => { if let Some(res) = res { if ctx.timer.is_none() { ctx.timer = Some(Instant::now_coarse()); } + let origin_applied_index = self.delegate.apply_state.get_applied_index(); + // flush with applied_index that TiFlash has flushed cache. + self.delegate.apply_state.set_applied_index(applied_index); ctx.prepare_for(&mut self.delegate); let mut result = VecDeque::new(); // If modified `truncated_state` in `try_compact_log`, the apply state should be @@ -4279,7 +4357,15 @@ where ctx.commit_opt(&mut self.delegate, true); } result.push_back(res); + self.delegate.apply_state.set_applied_index(origin_applied_index); ctx.finish_for(&mut self.delegate, result); + + info!( + "trigger compact log"; + "region_id" => self.delegate.region.get_id(), + "applied_index" => applied_index, + "origin applied_index" => origin_applied_index, + ); } } Err(e) => error!(?e; @@ -4368,12 +4454,14 @@ where Msg::CheckCompact { voter_replicated_index, voter_replicated_term, + applied_index, .. } => { self.check_pending_compact_log( apply_ctx, voter_replicated_index, voter_replicated_term, + applied_index, ); } } diff --git a/components/raftstore/src/store/fsm/peer.rs b/components/raftstore/src/store/fsm/peer.rs index 6acddde2257..1f1ba5bf2d5 100644 --- a/components/raftstore/src/store/fsm/peer.rs +++ b/components/raftstore/src/store/fsm/peer.rs @@ -2725,6 +2725,7 @@ where region_id: self.region_id(), voter_replicated_index, voter_replicated_term, + applied_index: self.fsm.peer.get_store().applied_index(), }, ) } @@ -3935,8 +3936,10 @@ where let total_cnt = self.fsm.peer.last_applying_idx - first_index; // the size of current CompactLog command can be ignored. let remain_cnt = self.fsm.peer.last_applying_idx - state.get_index() - 1; - self.fsm.peer.raft_log_size_hint = + if total_cnt != 0 { + self.fsm.peer.raft_log_size_hint = self.fsm.peer.raft_log_size_hint * remain_cnt / total_cnt; + } let compact_to = state.get_index() + 1; self.fsm.peer.schedule_raftlog_gc(self.ctx, compact_to); self.fsm.peer.last_compacted_idx = compact_to; diff --git a/engine_store_ffi/src/core/forward_raft/region.rs b/engine_store_ffi/src/core/forward_raft/region.rs index d91d9599dd4..5e476087bc8 100644 --- a/engine_store_ffi/src/core/forward_raft/region.rs +++ b/engine_store_ffi/src/core/forward_raft/region.rs @@ -116,4 +116,9 @@ impl ProxyForwarder { .access_cached_region_info_mut(region_id, f) .unwrap(); } + + pub fn get_compact_index_and_term(&self, region_id: u64) -> (u64, u64) { + return self.engine_store_server_helper.get_persisted_state(region_id); + } + } diff --git a/engine_store_ffi/src/ffi/mod.rs b/engine_store_ffi/src/ffi/mod.rs index 2354046f151..f1f0aea8932 100644 --- a/engine_store_ffi/src/ffi/mod.rs +++ b/engine_store_ffi/src/ffi/mod.rs @@ -6,7 +6,7 @@ pub use engine_tiflash::EngineStoreConfig; pub use proxy_ffi::{ basic_ffi_impls::*, domain_impls::*, encryption_impls::*, engine_store_helper_impls::*, interfaces::root::DB as interfaces_ffi, lock_cf_reader::*, raftstore_proxy, raftstore_proxy::*, - raftstore_proxy_helper_impls::*, read_index_helper, sst_reader_impls::*, + raftstore_proxy_helper_impls::*, read_index_helper, apply_router_helper, sst_reader_impls::*, }; pub use self::raftstore_proxy_engine::*; diff --git a/engine_store_ffi/src/observer.rs b/engine_store_ffi/src/observer.rs index 0a7f327feb2..2bdcc1b0c46 100644 --- a/engine_store_ffi/src/observer.rs +++ b/engine_store_ffi/src/observer.rs @@ -193,6 +193,12 @@ impl MessageObserver for TiFlashObserver fn on_raft_message(&self, msg: &RaftMessage) -> bool { self.forwarder.on_raft_message(msg) } + fn compact_log_in_queue(&self) -> bool { + false + } + fn get_compact_index_and_term(&self, region_id: u64, compact_index: u64, compact_term: u64) -> (u64, u64) { + return self.forwarder.get_compact_index_and_term(region_id); + } } impl PdTaskObserver for TiFlashObserver { diff --git a/proxy_ffi/src/apply_router_helper.rs b/proxy_ffi/src/apply_router_helper.rs new file mode 100644 index 00000000000..31698a0354f --- /dev/null +++ b/proxy_ffi/src/apply_router_helper.rs @@ -0,0 +1,51 @@ +use std::{ + future::Future, + time::{Duration, Instant}, +}; + +use engine_traits::{KvEngine, RaftEngine}; +use futures::executor::block_on; +use futures_util::{compat::Future01CompatExt, future::BoxFuture}; +use kvproto::{ + kvrpcpb::{ReadIndexRequest, ReadIndexResponse}, + raft_cmdpb::{CmdType, RaftCmdRequest, RaftRequestHeader, Request as RaftRequest}, +}; +use raftstore::{ + router::RaftStoreRouter, + store::{Callback, RaftCmdExtraOpts, RaftRouter, ReadResponse}, +}; +use tikv_util::{debug, error, future::paired_future_callback}; + +use super::utils::ArcNotifyWaker; +use raftstore::store::fsm::ApplyRouter; +use raftstore::store::fsm::apply::Msg; +use raftstore::store::fsm::ApplyTask; + +pub trait ApplyRouterHelper: Sync + Send{ + fn schedule_compact_log_task(&self, region_id: u64, compact_index: u64, compact_term: u64, applied_index: u64); +} + +pub struct ProxyApplyRouterHelper { + pub apply_router: std::sync::Mutex> +} + +impl ProxyApplyRouterHelper { + pub fn new(apply_router: ApplyRouter) -> Self { + Self { apply_router: std::sync::Mutex::new(apply_router.clone()) } + } +} + +impl ApplyRouterHelper for ProxyApplyRouterHelper { + fn schedule_compact_log_task(&self, region_id: u64, compact_index: u64, compact_term: u64, applied_index: u64) { + self.apply_router + .lock() + .unwrap() + .schedule_task(region_id, ApplyTask::CheckCompact { + region_id: region_id, + voter_replicated_index: compact_index, + voter_replicated_term: compact_term, + applied_index: applied_index, + }) + } +} + diff --git a/proxy_ffi/src/engine_store_helper_impls.rs b/proxy_ffi/src/engine_store_helper_impls.rs index a010e39ee30..70b59e61be6 100644 --- a/proxy_ffi/src/engine_store_helper_impls.rs +++ b/proxy_ffi/src/engine_store_helper_impls.rs @@ -329,6 +329,14 @@ impl EngineStoreServerHelper { debug_assert!(self.fn_fast_add_peer.is_some()); unsafe { (self.fn_fast_add_peer.into_inner())(self.inner, region_id, new_peer_id) } } + + pub fn get_persisted_state(&self, region_id: u64) -> (u64, u64) { + debug_assert!(self.fn_get_flushed_state.is_some()); + unsafe { + let flushed_state = (self.fn_get_flushed_state.into_inner())(self.inner, region_id); + return (flushed_state.applied_index, flushed_state.applied_term); + } + } } // PageStorage specific diff --git a/proxy_ffi/src/interfaces.rs b/proxy_ffi/src/interfaces.rs index 84cee67b158..b41537a4f8f 100644 --- a/proxy_ffi/src/interfaces.rs +++ b/proxy_ffi/src/interfaces.rs @@ -284,6 +284,12 @@ pub mod root { } #[repr(C)] #[derive(Debug)] + pub struct FlushedState { + pub applied_index: u64, + pub applied_term: u64, + } + #[repr(C)] + #[derive(Debug)] pub struct RaftStoreProxyFFIHelper { pub proxy_ptr: root::DB::RaftStoreProxyPtr, pub fn_handle_get_proxy_status: ::std::option::Option< @@ -383,6 +389,15 @@ pub mod root { error_msg: *mut root::DB::RawCppStringPtr, ) -> root::DB::KVGetStatus, >, + pub fn_notify_compact_log: ::std::option::Option< + unsafe extern "C" fn( + arg1: root::DB::RaftStoreProxyPtr, + arg2: u64, + arg3: u64, + arg4: u64, + arg5: u64, + ), + >, } #[repr(C)] #[derive(Debug)] @@ -584,6 +599,12 @@ pub mod root { new_peer_id: u64, ) -> root::DB::FastAddPeerRes, >, + pub fn_get_flushed_state: ::std::option::Option< + unsafe extern "C" fn( + arg1: *mut root::DB::EngineStoreServerWrap, + region_id: u64, + ) -> root::DB::FlushedState, + >, } pub const RAFT_STORE_PROXY_VERSION: u64 = 4990756589462826693; pub const RAFT_STORE_PROXY_MAGIC_NUMBER: u32 = 324508639; diff --git a/proxy_ffi/src/lib.rs b/proxy_ffi/src/lib.rs index 9964cf56413..5144c99c3f2 100644 --- a/proxy_ffi/src/lib.rs +++ b/proxy_ffi/src/lib.rs @@ -23,6 +23,7 @@ pub mod raftstore_proxy_helper_impls; pub mod read_index_helper; pub mod sst_reader_impls; pub mod utils; +pub mod apply_router_helper; pub use self::{ basic_ffi_impls::*, domain_impls::*, encryption_impls::*, engine_store_helper_impls::*, diff --git a/proxy_ffi/src/raftstore_proxy.rs b/proxy_ffi/src/raftstore_proxy.rs index de8333e983f..09223ecdd14 100644 --- a/proxy_ffi/src/raftstore_proxy.rs +++ b/proxy_ffi/src/raftstore_proxy.rs @@ -7,6 +7,8 @@ use std::sync::{ use encryption::DataKeyManager; +use crate::apply_router_helper; + use super::{ interfaces_ffi::{ConstRawVoidPtr, RaftProxyStatus, RaftStoreProxyPtr}, raftstore_proxy_helper_impls::*, @@ -20,6 +22,7 @@ pub struct RaftStoreProxy { key_manager: Option>, read_index_client: Option>, raftstore_proxy_engine: RwLock>, + apply_router_client: Option>, } impl RaftStoreProxy { @@ -28,12 +31,14 @@ impl RaftStoreProxy { key_manager: Option>, read_index_client: Option>, raftstore_proxy_engine: Option, + apply_router_client: Option>, ) -> Self { RaftStoreProxy { status, key_manager, read_index_client, raftstore_proxy_engine: RwLock::new(raftstore_proxy_engine), + apply_router_client, } } } @@ -93,6 +98,10 @@ impl RaftStoreProxyFFI for RaftStoreProxy { fn set_status(&mut self, s: RaftProxyStatus) { self.status.store(s as u8, Ordering::SeqCst); } + + fn maybe_apply_router_helper(&self) -> &Option> { + &self.apply_router_client + } } impl RaftStoreProxyPtr { diff --git a/proxy_ffi/src/raftstore_proxy_helper_impls.rs b/proxy_ffi/src/raftstore_proxy_helper_impls.rs index 97f5aaa9832..a5ab3fb52be 100644 --- a/proxy_ffi/src/raftstore_proxy_helper_impls.rs +++ b/proxy_ffi/src/raftstore_proxy_helper_impls.rs @@ -13,6 +13,8 @@ use std::{ use encryption::DataKeyManager; use kvproto::kvrpcpb; use protobuf::Message; +use tikv_util::store::region; +use raftstore::store::fsm::ApplyTask; use super::{ basic_ffi_impls::*, @@ -27,6 +29,7 @@ use super::{ read_index_helper, sst_reader_impls::*, utils, UnwrapExternCFunc, + apply_router_helper, }; impl Clone for RaftStoreProxyPtr { @@ -52,6 +55,7 @@ pub trait RaftStoreProxyFFI: Sync { // F: FnOnce(Result, String>); // fn set_kv_engine(&mut self, kv_engine: Option); // fn kv_engine(&self) -> &RwLock>; + fn maybe_apply_router_helper(&self) -> &Option>; } impl RaftStoreProxyFFIHelper { @@ -82,6 +86,7 @@ impl RaftStoreProxyFFIHelper { fn_make_timer_task: Some(ffi_make_timer_task), fn_poll_timer_task: Some(ffi_poll_timer_task), fn_get_region_local_state: Some(ffi_get_region_local_state), + fn_notify_compact_log: Some(ffi_notify_compact_log), } } } @@ -283,3 +288,29 @@ pub unsafe extern "C" fn ffi_poll_timer_task(task_ptr: RawVoidPtr, waker: RawVoi 0 } } + +pub extern "C" fn ffi_notify_compact_log( + proxy_ptr: RaftStoreProxyPtr, + region_id: u64, + compact_index: u64, + compact_term: u64, + applied_index: u64) { + assert!(!proxy_ptr.is_null()); + unsafe { + if proxy_ptr.as_ref().maybe_apply_router_helper().is_none() { + return; + } + } + unsafe { + proxy_ptr + .as_ref() + .maybe_apply_router_helper() + .as_ref() + .unwrap() + .schedule_compact_log_task( + region_id, + compact_index, + compact_term, + applied_index) + } +} diff --git a/proxy_server/src/run.rs b/proxy_server/src/run.rs index 6de038035d0..c786b16a04e 100644 --- a/proxy_server/src/run.rs +++ b/proxy_server/src/run.rs @@ -35,6 +35,7 @@ use engine_store_ffi::{ RaftStoreProxyFFIHelper, }, read_index_helper::ReadIndexClient, + apply_router_helper::ProxyApplyRouterHelper, RaftStoreProxy, RaftStoreProxyFFI, }, TiFlashEngine, @@ -148,6 +149,7 @@ pub fn run_impl( SysQuota::cpu_cores_quota() as usize * 2, ))), None, + Some(Box::new(ProxyApplyRouterHelper::new(tikv.system.as_ref().unwrap().apply_router()))), ); let proxy_ref = &proxy; @@ -256,6 +258,7 @@ fn run_impl_only_for_decryption( encryption_key_manager.clone(), Option::None, None, + None, ); let proxy_ref = &proxy; diff --git a/raftstore-proxy/ffi/src/RaftStoreProxyFFI/ProxyFFI.h b/raftstore-proxy/ffi/src/RaftStoreProxyFFI/ProxyFFI.h index 9684daa7c13..953f8e29556 100644 --- a/raftstore-proxy/ffi/src/RaftStoreProxyFFI/ProxyFFI.h +++ b/raftstore-proxy/ffi/src/RaftStoreProxyFFI/ProxyFFI.h @@ -192,6 +192,10 @@ struct FastAddPeerRes { CppStrWithView region; }; +struct FlushedState { + uint64_t applied_index; + uint64_t applied_term; +}; struct RaftStoreProxyFFIHelper { RaftStoreProxyPtr proxy_ptr; RaftProxyStatus (*fn_handle_get_proxy_status)(RaftStoreProxyPtr); @@ -221,6 +225,8 @@ struct RaftStoreProxyFFIHelper { KVGetStatus (*fn_get_region_local_state)(RaftStoreProxyPtr, uint64_t region_id, RawVoidPtr data, RawCppStringPtr *error_msg); + void (*fn_notify_compact_log)(RaftStoreProxyPtr, uint64_t, uint64_t, + uint64_t, uint64_t); }; struct PageStorageInterfaces { @@ -288,5 +294,6 @@ struct EngineStoreServerHelper { uint64_t leader_safe_ts); FastAddPeerRes (*fn_fast_add_peer)(EngineStoreServerWrap *, uint64_t region_id, uint64_t new_peer_id); + FlushedState (*fn_get_flushed_state)(EngineStoreServerWrap *, uint64_t region_id); }; } // namespace DB From 61ee64947e68e640b4db0b6aaaf51bddaea0b90c Mon Sep 17 00:00:00 2001 From: CalvinNeo Date: Mon, 29 May 2023 17:48:49 +0800 Subject: [PATCH 02/12] enhanced Signed-off-by: CalvinNeo --- components/raftstore/src/store/fsm/apply.rs | 25 ++++++++++++------- .../proxy_ffi/src/apply_router_helper.rs | 24 +++--------------- .../src/raftstore_proxy_helper_impls.rs | 2 -- .../ffi/src/RaftStoreProxyFFI/ProxyFFI.h | 4 +-- 4 files changed, 21 insertions(+), 34 deletions(-) diff --git a/components/raftstore/src/store/fsm/apply.rs b/components/raftstore/src/store/fsm/apply.rs index 661071217d5..6f2f5a482fd 100644 --- a/components/raftstore/src/store/fsm/apply.rs +++ b/components/raftstore/src/store/fsm/apply.rs @@ -1441,8 +1441,9 @@ where } if req.has_admin_request() { let request = req.get_admin_request(); - let cmd_type = request.get_cmd_type(); - if cmd_type == AdminCmdType::CompactLog { + // Filtering a CompactLog cmd means the underlying engine has not persisted to + // `compact_index` yet. + if request.get_cmd_type() == AdminCmdType::CompactLog { let compact_index = request.get_compact_log().get_compact_index(); let compact_term = request.get_compact_log().get_compact_term(); if compact_index > self.max_compact_index { @@ -3046,7 +3047,7 @@ where &mut self, voter_replicated_index: u64, voter_replicated_term: u64, - use_queue: bool, + handle_queue: bool, ) -> Result<(bool, Option>)> { PEER_ADMIN_CMD_COUNTER.compact.all.inc(); let first_index = entry_storage::first_index(&self.apply_state); @@ -3061,8 +3062,12 @@ where return Ok((false, None)); } - if !use_queue { + if !handle_queue { + // When `compact_log_in_queue` is set to false. let mut compact_index = voter_replicated_index; + // `compact_index` is reported by underlying engine, it can exceed the max + // recorded `compact_index` of this region. We will use the recorded + // one in this case. let mut compact_term = voter_replicated_term; if compact_index > self.max_compact_index { compact_index = self.max_compact_index; @@ -3228,12 +3233,14 @@ where } } } - if let Some((compact_index, compact_term)) = - ctx.host - .get_compact_index_and_term(self.region_id(), compact_index, compact_term) + + // TODO What's this? + if let Some((custom_compact_index, _custom_compact_term)) = ctx + .host + .get_compact_index_and_term(self.region_id(), compact_index, compact_term) { - if req.get_compact_log().get_compact_index() > compact_index - && compact_index > self.max_compact_index + if req.get_compact_log().get_compact_index() > self.max_compact_index + && custom_compact_index > self.max_compact_index { self.max_compact_index = req.get_compact_log().get_compact_index(); self.max_compact_term = req.get_compact_log().get_compact_term(); diff --git a/proxy_components/proxy_ffi/src/apply_router_helper.rs b/proxy_components/proxy_ffi/src/apply_router_helper.rs index b244ff8c5c1..bc2eb846566 100644 --- a/proxy_components/proxy_ffi/src/apply_router_helper.rs +++ b/proxy_components/proxy_ffi/src/apply_router_helper.rs @@ -1,25 +1,7 @@ -use std::{ - future::Future, - time::{Duration, Instant}, -}; +// Copyright 2023 TiKV Project Authors. Licensed under Apache-2.0. -use engine_traits::{KvEngine, RaftEngine}; -use futures::executor::block_on; -use futures_util::{compat::Future01CompatExt, future::BoxFuture}; -use kvproto::{ - kvrpcpb::{ReadIndexRequest, ReadIndexResponse}, - raft_cmdpb::{CmdType, RaftCmdRequest, RaftRequestHeader, Request as RaftRequest}, -}; -use raftstore::{ - router::RaftStoreRouter, - store::{ - fsm::{apply::Msg, ApplyRouter, ApplyTask}, - Callback, RaftCmdExtraOpts, RaftRouter, ReadResponse, - }, -}; -use tikv_util::{debug, error, future::paired_future_callback}; - -use super::utils::ArcNotifyWaker; +use engine_traits::KvEngine; +use raftstore::store::fsm::{ApplyRouter, ApplyTask}; pub trait ApplyRouterHelper: Sync + Send { fn schedule_compact_log_task( diff --git a/proxy_components/proxy_ffi/src/raftstore_proxy_helper_impls.rs b/proxy_components/proxy_ffi/src/raftstore_proxy_helper_impls.rs index f81f00e4ff8..e55ed6f872f 100644 --- a/proxy_components/proxy_ffi/src/raftstore_proxy_helper_impls.rs +++ b/proxy_components/proxy_ffi/src/raftstore_proxy_helper_impls.rs @@ -13,8 +13,6 @@ use std::{ use encryption::DataKeyManager; use kvproto::kvrpcpb; use protobuf::Message; -use raftstore::store::fsm::ApplyTask; -use tikv_util::store::region; use super::{ apply_router_helper, diff --git a/raftstore-proxy/ffi/src/RaftStoreProxyFFI/ProxyFFI.h b/raftstore-proxy/ffi/src/RaftStoreProxyFFI/ProxyFFI.h index e394dc516aa..40d39d8d750 100644 --- a/raftstore-proxy/ffi/src/RaftStoreProxyFFI/ProxyFFI.h +++ b/raftstore-proxy/ffi/src/RaftStoreProxyFFI/ProxyFFI.h @@ -240,8 +240,8 @@ struct RaftStoreProxyFFIHelper { KVGetStatus (*fn_get_region_local_state)(RaftStoreProxyPtr, uint64_t region_id, RawVoidPtr data, RawCppStringPtr *error_msg); - void (*fn_notify_compact_log)(RaftStoreProxyPtr, uint64_t, uint64_t, - uint64_t, uint64_t); + void (*fn_notify_compact_log)(RaftStoreProxyPtr, uint64_t region_id, uint64_t compact_index, + uint64_t compact_term, uint64_t applied_index); }; struct PageStorageInterfaces { From 6097c5f2f86a45072e737caacc8b2796ce5f3670 Mon Sep 17 00:00:00 2001 From: CalvinNeo Date: Wed, 28 Jun 2023 16:40:50 +0800 Subject: [PATCH 03/12] enhance some code Signed-off-by: CalvinNeo --- .../raftstore/src/coprocessor/dispatcher.rs | 24 ++- components/raftstore/src/coprocessor/mod.rs | 26 ++- components/raftstore/src/store/fsm/apply.rs | 159 ++++++++++-------- .../src/core/forward_raft/command.rs | 3 + .../engine_store_ffi/src/observer.rs | 15 +- .../mock_store/mock_engine_store_server.rs | 4 +- .../src/engine_store_helper_impls.rs | 6 +- proxy_components/proxy_ffi/src/interfaces.rs | 21 ++- .../ffi/src/RaftStoreProxyFFI/@version | 2 +- .../ffi/src/RaftStoreProxyFFI/ProxyFFI.h | 19 ++- 10 files changed, 167 insertions(+), 112 deletions(-) diff --git a/components/raftstore/src/coprocessor/dispatcher.rs b/components/raftstore/src/coprocessor/dispatcher.rs index 9294d01b522..d0e374c48ea 100644 --- a/components/raftstore/src/coprocessor/dispatcher.rs +++ b/components/raftstore/src/coprocessor/dispatcher.rs @@ -554,13 +554,20 @@ impl CoprocessorHost { } // (index, term) is for the applying entry. - pub fn pre_exec(&self, region: &Region, cmd: &RaftCmdRequest, index: u64, term: u64) -> bool { + pub fn pre_exec( + &self, + region: &Region, + cmd: &RaftCmdRequest, + index: u64, + term: u64, + apply_state: &RaftApplyState, + ) -> bool { let mut ctx = ObserverContext::new(region); if !cmd.has_admin_request() { let query = cmd.get_requests(); for observer in &self.registry.query_observers { let observer = observer.observer.inner(); - if observer.pre_exec_query(&mut ctx, query, index, term) { + if observer.pre_exec_query(&mut ctx, query, index, term, apply_state) { return true; } } @@ -569,7 +576,7 @@ impl CoprocessorHost { let admin = cmd.get_admin_request(); for observer in &self.registry.admin_observers { let observer = observer.observer.inner(); - if observer.pre_exec_admin(&mut ctx, admin, index, term) { + if observer.pre_exec_admin(&mut ctx, admin, index, term, apply_state) { return true; } } @@ -847,7 +854,7 @@ impl CoprocessorHost { } pub fn compact_log_in_queue(&self) -> bool { - for observer in &self.registry.message_observers { + for observer in &self.registry.region_change_observers { let observer = observer.observer.inner(); return observer.compact_log_in_queue(); } @@ -860,7 +867,7 @@ impl CoprocessorHost { compact_index: u64, compact_term: u64, ) -> Option<(u64, u64)> { - for observer in &self.registry.message_observers { + for observer in &self.registry.region_change_observers { let observer = observer.observer.inner(); let res = observer.get_compact_index_and_term(region_id, compact_index, compact_term); if res.is_some() { @@ -971,6 +978,7 @@ mod tests { _: &AdminRequest, _: u64, _: u64, + _: &RaftApplyState, ) -> bool { self.called .fetch_add(ObserverIndex::PreExecAdmin as usize, Ordering::SeqCst); @@ -1026,6 +1034,7 @@ mod tests { _: &[Request], _: u64, _: u64, + _: &RaftApplyState, ) -> bool { self.called .fetch_add(ObserverIndex::PreExecQuery as usize, Ordering::SeqCst); @@ -1286,14 +1295,15 @@ mod tests { assert_all!([&ob.called], &[index]); let mut query_req = RaftCmdRequest::default(); + let apply_state = RaftApplyState::default(); query_req.set_requests(vec![Request::default()].into()); - host.pre_exec(®ion, &query_req, 0, 0); + host.pre_exec(®ion, &query_req, 0, 0, &apply_state); index += ObserverIndex::PreExecQuery as usize; assert_all!([&ob.called], &[index]); let mut admin_req = RaftCmdRequest::default(); admin_req.set_admin_request(AdminRequest::default()); - host.pre_exec(®ion, &admin_req, 0, 0); + host.pre_exec(®ion, &admin_req, 0, 0, &apply_state); index += ObserverIndex::PreExecAdmin as usize; assert_all!([&ob.called], &[index]); diff --git a/components/raftstore/src/coprocessor/mod.rs b/components/raftstore/src/coprocessor/mod.rs index 1c6d001a21e..5efc0d9645c 100644 --- a/components/raftstore/src/coprocessor/mod.rs +++ b/components/raftstore/src/coprocessor/mod.rs @@ -113,6 +113,7 @@ pub trait AdminObserver: Coprocessor { _: &AdminRequest, _: u64, _: u64, + _: &RaftApplyState, ) -> bool { false } @@ -153,7 +154,14 @@ pub trait QueryObserver: Coprocessor { /// Hook before exec write request, returns whether we should skip this /// write. - fn pre_exec_query(&self, _: &mut ObserverContext<'_>, _: &[Request], _: u64, _: u64) -> bool { + fn pre_exec_query( + &self, + _: &mut ObserverContext<'_>, + _: &[Request], + _: u64, + _: u64, + _: &RaftApplyState, + ) -> bool { false } @@ -337,18 +345,13 @@ pub trait RegionChangeObserver: Coprocessor { fn pre_write_apply_state(&self, _: &mut ObserverContext<'_>) -> bool { true } -} - -pub trait MessageObserver: Coprocessor { - /// Returns false if the message should not be stepped later. - fn on_raft_message(&self, _: &RaftMessage) -> bool { - true - } /// Returns true if need to get compact log index from pending queue. fn compact_log_in_queue(&self) -> bool { true } + + /// Returns the maximum index the underlying engine can compact. fn get_compact_index_and_term( &self, region_id: u64, @@ -359,6 +362,13 @@ pub trait MessageObserver: Coprocessor { } } +pub trait MessageObserver: Coprocessor { + /// Returns false if the message should not be stepped later. + fn on_raft_message(&self, _: &RaftMessage) -> bool { + true + } +} + #[derive(Clone, Debug, Default)] pub struct Cmd { pub index: u64, diff --git a/components/raftstore/src/store/fsm/apply.rs b/components/raftstore/src/store/fsm/apply.rs index 6f2f5a482fd..e72c20a66b4 100644 --- a/components/raftstore/src/store/fsm/apply.rs +++ b/components/raftstore/src/store/fsm/apply.rs @@ -999,9 +999,13 @@ where /// The counter of pending request snapshots. See more in `Peer`. pending_request_snapshot_count: Arc, - /// The max compact index from leader + /// Keep track of the max compact index from leader. + /// This index may not be exactly accurate, since a restart may have these + /// values set to 0. However, it can always be advanced, since + /// `CompactLog` will try to update it. max_compact_index: u64, max_compact_term: u64, + /// Indicates the peer is in merging, if that compact log won't be /// performed. is_merging: bool, @@ -1432,72 +1436,78 @@ where // E.g. `RaftApplyState` must not be changed. let mut origin_epoch = None; - let (resp, exec_result) = if ctx.host.pre_exec(&self.region, &req, index, term) { - // One of the observers want to filter execution of the command. - let mut resp = RaftCmdResponse::default(); - if !req.get_header().get_uuid().is_empty() { - let uuid = req.get_header().get_uuid().to_vec(); - resp.mut_header().set_uuid(uuid); - } - if req.has_admin_request() { - let request = req.get_admin_request(); - // Filtering a CompactLog cmd means the underlying engine has not persisted to - // `compact_index` yet. - if request.get_cmd_type() == AdminCmdType::CompactLog { - let compact_index = request.get_compact_log().get_compact_index(); - let compact_term = request.get_compact_log().get_compact_term(); - if compact_index > self.max_compact_index { - self.max_compact_index = compact_index; - self.max_compact_term = compact_term; - } + let (resp, exec_result) = + if ctx + .host + .pre_exec(&self.region, &req, index, term, &self.apply_state) + { + // One of the observers want to filter execution of the command. + let mut resp = RaftCmdResponse::default(); + if !req.get_header().get_uuid().is_empty() { + let uuid = req.get_header().get_uuid().to_vec(); + resp.mut_header().set_uuid(uuid); } - } - (resp, ApplyResult::None) - } else { - ctx.exec_log_index = index; - ctx.exec_log_term = term; - ctx.kv_wb_mut().set_save_point(); - let (resp, exec_result) = match self.exec_raft_cmd(ctx, &req) { - Ok(a) => { - ctx.kv_wb_mut().pop_save_point().unwrap(); - if req.has_admin_request() { - origin_epoch = Some(self.region.get_region_epoch().clone()); + if req.has_admin_request() { + let request = req.get_admin_request(); + // Filtering a CompactLog cmd means the underlying engine has not persisted to + // `compact_index` yet. + // We will keep track of the index so later we can try to compact to a index not + // greater than `max_compact_index`. + if request.get_cmd_type() == AdminCmdType::CompactLog { + let compact_index = request.get_compact_log().get_compact_index(); + let compact_term = request.get_compact_log().get_compact_term(); + if compact_index > self.max_compact_index { + self.max_compact_index = compact_index; + self.max_compact_term = compact_term; + } } - a } - Err(e) => { - // clear dirty values. - ctx.kv_wb_mut().rollback_to_save_point().unwrap(); - match e { - Error::EpochNotMatch(..) => debug!( - "epoch not match"; - "region_id" => self.region_id(), - "peer_id" => self.id(), - "err" => ?e - ), - Error::FlashbackInProgress(..) => debug!( - "flashback is in process"; - "region_id" => self.region_id(), - "peer_id" => self.id(), - "err" => ?e - ), - Error::FlashbackNotPrepared(..) => debug!( - "flashback is not prepared"; - "region_id" => self.region_id(), - "peer_id" => self.id(), - "err" => ?e - ), - _ => error!(?e; - "execute raft command"; - "region_id" => self.region_id(), - "peer_id" => self.id(), - ), + (resp, ApplyResult::None) + } else { + ctx.exec_log_index = index; + ctx.exec_log_term = term; + ctx.kv_wb_mut().set_save_point(); + let (resp, exec_result) = match self.exec_raft_cmd(ctx, &req) { + Ok(a) => { + ctx.kv_wb_mut().pop_save_point().unwrap(); + if req.has_admin_request() { + origin_epoch = Some(self.region.get_region_epoch().clone()); + } + a } - (cmd_resp::new_error(e), ApplyResult::None) - } + Err(e) => { + // clear dirty values. + ctx.kv_wb_mut().rollback_to_save_point().unwrap(); + match e { + Error::EpochNotMatch(..) => debug!( + "epoch not match"; + "region_id" => self.region_id(), + "peer_id" => self.id(), + "err" => ?e + ), + Error::FlashbackInProgress(..) => debug!( + "flashback is in process"; + "region_id" => self.region_id(), + "peer_id" => self.id(), + "err" => ?e + ), + Error::FlashbackNotPrepared(..) => debug!( + "flashback is not prepared"; + "region_id" => self.region_id(), + "peer_id" => self.id(), + "err" => ?e + ), + _ => error!(?e; + "execute raft command"; + "region_id" => self.region_id(), + "peer_id" => self.id(), + ), + } + (cmd_resp::new_error(e), ApplyResult::None) + } + }; + (resp, exec_result) }; - (resp, exec_result) - }; let cmd = Cmd::new(index, term, req, resp); if let ApplyResult::WaitMergeSource(_) = exec_result { @@ -3064,10 +3074,11 @@ where if !handle_queue { // When `compact_log_in_queue` is set to false. + // It it a compact request from underlying engine. let mut compact_index = voter_replicated_index; - // `compact_index` is reported by underlying engine, it can exceed the max - // recorded `compact_index` of this region. We will use the recorded - // one in this case. + // `compact_index` is reported by underlying engine, it may be greater than the + // recorded `max_compact_index`. We will use the smaller one of this + // two. let mut compact_term = voter_replicated_term; if compact_index > self.max_compact_index { compact_index = self.max_compact_index; @@ -3234,16 +3245,19 @@ where } } - // TODO What's this? - if let Some((custom_compact_index, _custom_compact_term)) = ctx + if let Some((custom_compact_index, custom_compact_term)) = ctx .host .get_compact_index_and_term(self.region_id(), compact_index, compact_term) { - if req.get_compact_log().get_compact_index() > self.max_compact_index - && custom_compact_index > self.max_compact_index - { - self.max_compact_index = req.get_compact_log().get_compact_index(); - self.max_compact_term = req.get_compact_log().get_compact_term(); + if custom_compact_index < compact_index { + // If the underlying engine has not compact to expected yet, + // We will compact to what the underlying engine tells which is a best effort. + compact_index = custom_compact_index; + compact_term = custom_compact_term; + } + if compact_index > self.max_compact_index { + self.max_compact_index = compact_index; + self.max_compact_term = compact_term; } } if compact_index < first_index { @@ -5921,6 +5935,7 @@ mod tests { req: &AdminRequest, _: u64, _: u64, + _: &RaftApplyState, ) -> bool { let cmd_type = req.get_cmd_type(); if cmd_type == AdminCmdType::CompactLog diff --git a/proxy_components/engine_store_ffi/src/core/forward_raft/command.rs b/proxy_components/engine_store_ffi/src/core/forward_raft/command.rs index 985617289b1..c3e10383c00 100644 --- a/proxy_components/engine_store_ffi/src/core/forward_raft/command.rs +++ b/proxy_components/engine_store_ffi/src/core/forward_raft/command.rs @@ -75,6 +75,7 @@ impl ProxyForwarder { req: &AdminRequest, index: u64, term: u64, + apply_state: &RaftApplyState, ) -> bool { match req.get_cmd_type() { AdminCmdType::CompactLog => { @@ -84,6 +85,8 @@ impl ProxyForwarder { false, index, term, + apply_state.get_truncated_state().get_index(), + apply_state.get_truncated_state().get_term(), ) { info!("can't flush data, filter CompactLog"; "region_id" => ?ob_region.get_id(), diff --git a/proxy_components/engine_store_ffi/src/observer.rs b/proxy_components/engine_store_ffi/src/observer.rs index cdd97df96d1..78c045d9372 100644 --- a/proxy_components/engine_store_ffi/src/observer.rs +++ b/proxy_components/engine_store_ffi/src/observer.rs @@ -110,9 +110,10 @@ impl AdminObserver for TiFlashObserver bool { self.forwarder - .pre_exec_admin(ob_ctx.region(), req, index, term) + .pre_exec_admin(ob_ctx.region(), req, index, term, apply_state) } fn post_exec_admin( @@ -187,15 +188,11 @@ impl RegionChangeObserver for TiFlashObs fn pre_write_apply_state(&self, ob_ctx: &mut ObserverContext<'_>) -> bool { self.forwarder.pre_write_apply_state(ob_ctx.region()) } -} -impl MessageObserver for TiFlashObserver { - fn on_raft_message(&self, msg: &RaftMessage) -> bool { - self.forwarder.on_raft_message(msg) - } fn compact_log_in_queue(&self) -> bool { false } + fn get_compact_index_and_term( &self, region_id: u64, @@ -207,6 +204,12 @@ impl MessageObserver for TiFlashObserver } } +impl MessageObserver for TiFlashObserver { + fn on_raft_message(&self, msg: &RaftMessage) -> bool { + self.forwarder.on_raft_message(msg) + } +} + impl PdTaskObserver for TiFlashObserver { fn on_compute_engine_size(&self, store_size: &mut Option) { self.forwarder.on_compute_engine_size(store_size) diff --git a/proxy_components/mock-engine-store/src/mock_store/mock_engine_store_server.rs b/proxy_components/mock-engine-store/src/mock_store/mock_engine_store_server.rs index 5759475f840..3342ea64853 100644 --- a/proxy_components/mock-engine-store/src/mock_store/mock_engine_store_server.rs +++ b/proxy_components/mock-engine-store/src/mock_store/mock_engine_store_server.rs @@ -339,9 +339,11 @@ extern "C" fn ffi_need_flush_data( unsafe extern "C" fn ffi_try_flush_data( arg1: *mut interfaces_ffi::EngineStoreServerWrap, region_id: u64, - _try_until_succeed: u8, + _flush_pattern: u8, index: u64, term: u64, + _truncated_index: u64, + _truncated_term: u64, ) -> u8 { let store = into_engine_store_server_wrap(arg1); let kvstore = &mut (*store.engine_store_server).kvstore; diff --git a/proxy_components/proxy_ffi/src/engine_store_helper_impls.rs b/proxy_components/proxy_ffi/src/engine_store_helper_impls.rs index 6eac03bdf75..686a1f75688 100644 --- a/proxy_components/proxy_ffi/src/engine_store_helper_impls.rs +++ b/proxy_components/proxy_ffi/src/engine_store_helper_impls.rs @@ -156,6 +156,8 @@ impl EngineStoreServerHelper { try_until_succeed: bool, index: u64, term: u64, + truncated_index: u64, + truncated_term: u64, ) -> bool { debug_assert!(self.fn_try_flush_data.is_some()); unsafe { @@ -170,6 +172,8 @@ impl EngineStoreServerHelper { }, index, term, + truncated_index, + truncated_term, ) != 0 } } @@ -333,7 +337,7 @@ impl EngineStoreServerHelper { pub fn get_persisted_state(&self, region_id: u64) -> (u64, u64) { debug_assert!(self.fn_get_flushed_state.is_some()); unsafe { - let flushed_state = (self.fn_get_flushed_state.into_inner())(self.inner, region_id); + let flushed_state = (self.fn_get_flushed_state.into_inner())(self.inner, region_id, 1); return (flushed_state.applied_index, flushed_state.applied_term); } } diff --git a/proxy_components/proxy_ffi/src/interfaces.rs b/proxy_components/proxy_ffi/src/interfaces.rs index baf0bdecd3c..8b4c0fccf06 100644 --- a/proxy_components/proxy_ffi/src/interfaces.rs +++ b/proxy_components/proxy_ffi/src/interfaces.rs @@ -420,10 +420,10 @@ pub mod root { pub fn_notify_compact_log: ::std::option::Option< unsafe extern "C" fn( arg1: root::DB::RaftStoreProxyPtr, - arg2: u64, - arg3: u64, - arg4: u64, - arg5: u64, + region_id: u64, + compact_index: u64, + compact_term: u64, + applied_index: u64, ), >, } @@ -517,10 +517,12 @@ pub mod root { pub fn_try_flush_data: ::std::option::Option< unsafe extern "C" fn( arg1: *mut root::DB::EngineStoreServerWrap, - arg2: u64, - arg3: u8, - arg4: u64, - arg5: u64, + region_id: u64, + flush_pattern: u8, + index: u64, + term: u64, + truncated_index: u64, + truncated_term: u64, ) -> u8, >, pub fn_atomic_update_proxy: ::std::option::Option< @@ -631,10 +633,11 @@ pub mod root { unsafe extern "C" fn( arg1: *mut root::DB::EngineStoreServerWrap, region_id: u64, + acquire_lock: u8, ) -> root::DB::FlushedState, >, } - pub const RAFT_STORE_PROXY_VERSION: u64 = 3617226644007633432; + pub const RAFT_STORE_PROXY_VERSION: u64 = 12138733948555436956; pub const RAFT_STORE_PROXY_MAGIC_NUMBER: u32 = 324508639; } } diff --git a/raftstore-proxy/ffi/src/RaftStoreProxyFFI/@version b/raftstore-proxy/ffi/src/RaftStoreProxyFFI/@version index 98d5effe612..536dcf0d27b 100644 --- a/raftstore-proxy/ffi/src/RaftStoreProxyFFI/@version +++ b/raftstore-proxy/ffi/src/RaftStoreProxyFFI/@version @@ -1,3 +1,3 @@ #pragma once #include -namespace DB { constexpr uint64_t RAFT_STORE_PROXY_VERSION = 3617226644007633432ull; } \ No newline at end of file +namespace DB { constexpr uint64_t RAFT_STORE_PROXY_VERSION = 12138733948555436956ull; } \ No newline at end of file diff --git a/raftstore-proxy/ffi/src/RaftStoreProxyFFI/ProxyFFI.h b/raftstore-proxy/ffi/src/RaftStoreProxyFFI/ProxyFFI.h index 40d39d8d750..8bf9d3d0394 100644 --- a/raftstore-proxy/ffi/src/RaftStoreProxyFFI/ProxyFFI.h +++ b/raftstore-proxy/ffi/src/RaftStoreProxyFFI/ProxyFFI.h @@ -208,8 +208,8 @@ struct FastAddPeerRes { }; struct FlushedState { - uint64_t applied_index; - uint64_t applied_term; + uint64_t applied_index; + uint64_t applied_term; }; struct RaftStoreProxyFFIHelper { RaftStoreProxyPtr proxy_ptr; @@ -240,8 +240,9 @@ struct RaftStoreProxyFFIHelper { KVGetStatus (*fn_get_region_local_state)(RaftStoreProxyPtr, uint64_t region_id, RawVoidPtr data, RawCppStringPtr *error_msg); - void (*fn_notify_compact_log)(RaftStoreProxyPtr, uint64_t region_id, uint64_t compact_index, - uint64_t compact_term, uint64_t applied_index); + void (*fn_notify_compact_log)(RaftStoreProxyPtr, uint64_t region_id, + uint64_t compact_index, uint64_t compact_term, + uint64_t applied_index); }; struct PageStorageInterfaces { @@ -278,8 +279,10 @@ struct EngineStoreServerHelper { BaseBuffView, BaseBuffView, RaftCmdHeader); uint8_t (*fn_need_flush_data)(EngineStoreServerWrap *, uint64_t); - uint8_t (*fn_try_flush_data)(EngineStoreServerWrap *, uint64_t, uint8_t, - uint64_t, uint64_t); + uint8_t (*fn_try_flush_data)(EngineStoreServerWrap *, uint64_t region_id, + uint8_t flush_pattern, uint64_t index, + uint64_t term, uint64_t truncated_index, + uint64_t truncated_term); void (*fn_atomic_update_proxy)(EngineStoreServerWrap *, RaftStoreProxyFFIHelper *); void (*fn_handle_destroy)(EngineStoreServerWrap *, uint64_t); @@ -309,6 +312,8 @@ struct EngineStoreServerHelper { uint64_t leader_safe_ts); FastAddPeerRes (*fn_fast_add_peer)(EngineStoreServerWrap *, uint64_t region_id, uint64_t new_peer_id); - FlushedState (*fn_get_flushed_state)(EngineStoreServerWrap *, uint64_t region_id); + FlushedState (*fn_get_flushed_state)(EngineStoreServerWrap *, + uint64_t region_id, + uint8_t acquire_lock); }; } // namespace DB From e04d1a81bde78d5cc1cf98b8b3970bfe35db0987 Mon Sep 17 00:00:00 2001 From: CalvinNeo Date: Thu, 6 Jul 2023 11:03:11 +0800 Subject: [PATCH 04/12] Merge proxy master --- Cargo.lock | 56 ++-- Cargo.toml | 4 +- components/raftstore/src/coprocessor/mod.rs | 6 +- components/raftstore/src/store/fsm/apply.rs | 4 +- .../src/core/forward_raft/command.rs | 81 +++++ .../src/core/forward_raft/region.rs | 4 +- .../engine_store_ffi/src/core/forwarder.rs | 6 + .../engine_store_ffi/src/observer.rs | 3 + .../src/mock_cluster/cluster_ext.rs | 19 ++ .../src/mock_cluster/v1/cluster.rs | 5 +- .../src/mock_cluster/v1/node.rs | 2 + .../src/mock_cluster/v1/server.rs | 2 + .../src/mock_cluster/v1/util.rs | 1 + .../mock_store/mock_engine_store_server.rs | 1 + proxy_components/proxy_ffi/Cargo.toml | 3 + proxy_components/proxy_ffi/src/interfaces.rs | 23 +- .../proxy_ffi/src/raftstore_proxy.rs | 186 ++++++++++- .../src/raftstore_proxy_helper_impls.rs | 27 +- .../sst_reader_dispatcher.rs | 4 +- proxy_components/proxy_server/src/run.rs | 10 + proxy_components/proxy_server/src/util.rs | 31 +- proxy_scripts/ci_check.sh | 1 + proxy_scripts/clippy.sh | 0 proxy_tests/Cargo.toml | 1 + proxy_tests/proxy/mod.rs | 1 + proxy_tests/proxy/shared/ingest.rs | 49 +++ proxy_tests/proxy/utils/mod.rs | 1 + proxy_tests/proxy/utils/sst.rs | 184 +++++++++++ proxy_tests/proxy/utils/v1.rs | 52 +++ .../proxy/v2_compat/cluster_raftstore_ver.rs | 300 ++++++++++++++++++ proxy_tests/proxy/v2_compat/mod.rs | 1 + .../proxy/v2_compat/tablet_snapshot.rs | 2 +- .../ffi/src/RaftStoreProxyFFI/@version | 2 +- .../ffi/src/RaftStoreProxyFFI/ProxyFFI.h | 20 ++ 34 files changed, 1048 insertions(+), 44 deletions(-) mode change 100644 => 100755 proxy_scripts/clippy.sh create mode 100644 proxy_tests/proxy/utils/sst.rs create mode 100644 proxy_tests/proxy/v2_compat/cluster_raftstore_ver.rs diff --git a/Cargo.lock b/Cargo.lock index a0934490dce..a42d38acf46 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2067,11 +2067,10 @@ checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b" [[package]] name = "form_urlencoded" -version = "1.0.1" +version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5fc25a87fa4fd2094bffb06925852034d90a17f0d1e05197d4956d3555752191" +checksum = "a62bc1cf6f830c2ec14a513a9fb124d0a213a629668a4186f329db21fe045652" dependencies = [ - "matches", "percent-encoding", ] @@ -2674,11 +2673,10 @@ checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39" [[package]] name = "idna" -version = "0.2.0" +version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "02e2673c30ee86b5b96a9cb52ad15718aa1f966f5ab9ad54a8b95d5ca33120a9" +checksum = "7d20d6b07bfbc108882d88ed8e37d39636dcc260e15e30c45e6ba089610b917c" dependencies = [ - "matches", "unicode-bidi", "unicode-normalization", ] @@ -3961,9 +3959,9 @@ checksum = "19b17cddbe7ec3f8bc800887bab5e717348c95ea2ca0b1bf0837fb964dc67099" [[package]] name = "percent-encoding" -version = "2.1.0" +version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d4fd5641d01c8f18a23da7b6fe29298ff4b55afcccdf78973b24cf3175fee32e" +checksum = "9b2a4787296e9989611394c33f193f676704af1686e70b8f8033ab5ba9a35a94" [[package]] name = "perfcnt" @@ -4443,8 +4441,10 @@ dependencies = [ "keys", "kvproto", "lazy_static", + "pd_client", "protobuf", "raftstore", + "reqwest", "rocksdb", "slog", "slog-global", @@ -4452,6 +4452,7 @@ dependencies = [ "tokio", "tokio-timer", "tracker", + "url", ] [[package]] @@ -4563,6 +4564,7 @@ dependencies = [ "futures 0.3.15", "grpcio", "grpcio-health", + "hex 0.4.2", "hyper", "keys", "kvproto", @@ -6123,9 +6125,8 @@ checksum = "20518fe4a4c9acf048008599e464deb21beeae3d3578418951a189c235a7a9a8" [[package]] name = "sysinfo" -version = "0.26.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ade661fa5e048ada64ad7901713301c21d2dbc5b65ee7967de8826c111452960" +version = "0.26.9" +source = "git+https://github.com/tikv/sysinfo?branch=0.26-fix-cpu#5a1bcf08816979624ef2ad79cfb896de432a9501" dependencies = [ "cfg-if 1.0.0", "core-foundation-sys", @@ -7059,6 +7060,21 @@ dependencies = [ "serde_json", ] +[[package]] +name = "tinyvec" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87cc5ceb3875bb20c2890005a4e226a4651264a5c75edb2421b52861a0a0cb50" +dependencies = [ + "tinyvec_macros", +] + +[[package]] +name = "tinyvec_macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" + [[package]] name = "tipb" version = "0.0.1" @@ -7413,12 +7429,9 @@ checksum = "eeba86d422ce181a719445e51872fa30f1f7413b62becb52e95ec91aa262d85c" [[package]] name = "unicode-bidi" -version = "0.3.4" +version = "0.3.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49f2bd0c6468a8230e1db229cff8029217cf623c767ea5d60bfbd42729ea54d5" -dependencies = [ - "matches", -] +checksum = "92888ba5573ff080736b3648696b70cafad7d250551175acbaa4e0385b3e1460" [[package]] name = "unicode-ident" @@ -7428,11 +7441,11 @@ checksum = "6ceab39d59e4c9499d4e5a8ee0e2735b891bb7308ac83dfb4e80cad195c9f6f3" [[package]] name = "unicode-normalization" -version = "0.1.12" +version = "0.1.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5479532badd04e128284890390c1e876ef7a993d0570b3597ae43dfa1d59afa4" +checksum = "5c5713f0fc4b5db668a2ac63cdb7bb4469d8c9fed047b1d0292cc7b0ce2ba921" dependencies = [ - "smallvec", + "tinyvec", ] [[package]] @@ -7455,13 +7468,12 @@ checksum = "a156c684c91ea7d62626509bce3cb4e1d9ed5c4d978f7b4352658f96a4c26b4a" [[package]] name = "url" -version = "2.2.2" +version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a507c383b2d33b5fc35d1861e77e6b383d158b2da5e14fe51b83dfedf6fd578c" +checksum = "50bff7831e19200a85b17131d085c25d7811bc4e186efdaf54bbd132994a88cb" dependencies = [ "form_urlencoded", "idna", - "matches", "percent-encoding", "serde", ] diff --git a/Cargo.toml b/Cargo.toml index 42725dae955..a0cbfe23084 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -222,6 +222,8 @@ cmake = { git = "https://github.com/rust-lang/cmake-rs" } # remove this when https://github.com/rust-lang/backtrace-rs/pull/503 is merged. backtrace = { git = 'https://github.com/hehechen/backtrace-rs', branch = "v0.3.61" } +sysinfo ={ git = "https://github.com/tikv/sysinfo", branch = "0.26-fix-cpu" } + [target.'cfg(target_os = "linux")'.dependencies] procinfo = { git = "https://github.com/tikv/procinfo-rs", rev = "6599eb9dca74229b2c1fcc44118bef7eff127128" } # When you modify TiKV cooperatively with kvproto, this will be useful to submit the PR to TiKV and the PR to @@ -424,7 +426,7 @@ opt-level = 1 [profile.dev] opt-level = 0 -debug = 0 +debug = false codegen-units = 4 lto = false incremental = true diff --git a/components/raftstore/src/coprocessor/mod.rs b/components/raftstore/src/coprocessor/mod.rs index 5efc0d9645c..f50d0c85c92 100644 --- a/components/raftstore/src/coprocessor/mod.rs +++ b/components/raftstore/src/coprocessor/mod.rs @@ -354,9 +354,9 @@ pub trait RegionChangeObserver: Coprocessor { /// Returns the maximum index the underlying engine can compact. fn get_compact_index_and_term( &self, - region_id: u64, - compact_index: u64, - compact_term: u64, + _: u64, + _: u64, + _: u64, ) -> Option<(u64, u64)> { None } diff --git a/components/raftstore/src/store/fsm/apply.rs b/components/raftstore/src/store/fsm/apply.rs index e72c20a66b4..b8884c682c9 100644 --- a/components/raftstore/src/store/fsm/apply.rs +++ b/components/raftstore/src/store/fsm/apply.rs @@ -3927,8 +3927,8 @@ where } => { write!( f, - "[region {}] check compact, voter_replicated_index: {}, voter_replicated_term: {}", - region_id, voter_replicated_index, voter_replicated_term + "[region {}] check compact, voter_replicated_index: {}, voter_replicated_term: {}, applied_index: {}", + region_id, voter_replicated_index, voter_replicated_term, applied_index ) } } diff --git a/proxy_components/engine_store_ffi/src/core/forward_raft/command.rs b/proxy_components/engine_store_ffi/src/core/forward_raft/command.rs index c3e10383c00..aa414208637 100644 --- a/proxy_components/engine_store_ffi/src/core/forward_raft/command.rs +++ b/proxy_components/engine_store_ffi/src/core/forward_raft/command.rs @@ -1,6 +1,86 @@ // Copyright 2022 TiKV Project Authors. Licensed under Apache-2.0. +use encryption::DataKeyManager; +use proxy_ffi::snapshot_reader_impls::{sst_file_reader::SSTFileReader, LockCFFileReader}; + use crate::core::{common::*, ProxyForwarder}; +pub fn get_first_key( + path: &str, + cf: ColumnFamilyType, + key_manager: Option>, +) -> Vec { + unsafe { + if cf == ColumnFamilyType::Lock { + let mut reader = LockCFFileReader::ffi_get_cf_file_reader(path, key_manager.as_ref()); + reader.as_mut_sst_lock().ffi_key().to_slice().to_vec() + } else { + let mut reader = SSTFileReader::ffi_get_cf_file_reader(path, key_manager); + reader.as_mut_sst_other().ffi_key().to_slice().to_vec() + } + } +} + +pub fn sort_sst_by_start_key( + ssts: Vec<(PathBuf, ColumnFamilyType)>, + key_manager: Option>, +) -> Vec<(PathBuf, ColumnFamilyType)> { + let mut sw: Vec<(PathBuf, ColumnFamilyType)> = vec![]; + let mut sd: Vec<(PathBuf, ColumnFamilyType)> = vec![]; + let mut sl: Vec<(PathBuf, ColumnFamilyType)> = vec![]; + + for (p, c) in ssts.into_iter() { + match c { + ColumnFamilyType::Default => sd.push((p, c)), + ColumnFamilyType::Write => sw.push((p, c)), + ColumnFamilyType::Lock => sl.push((p, c)), + }; + } + + sw.sort_by(|a, b| { + let fk1 = get_first_key( + a.0.to_str().unwrap(), + ColumnFamilyType::Write, + key_manager.clone(), + ); + let fk2 = get_first_key( + b.0.to_str().unwrap(), + ColumnFamilyType::Write, + key_manager.clone(), + ); + fk1.cmp(&fk2) + }); + sd.sort_by(|a, b| { + let fk1 = get_first_key( + a.0.to_str().unwrap(), + ColumnFamilyType::Default, + key_manager.clone(), + ); + let fk2 = get_first_key( + b.0.to_str().unwrap(), + ColumnFamilyType::Default, + key_manager.clone(), + ); + fk1.cmp(&fk2) + }); + sl.sort_by(|a, b| { + let fk1 = get_first_key( + a.0.to_str().unwrap(), + ColumnFamilyType::Lock, + key_manager.clone(), + ); + let fk2 = get_first_key( + b.0.to_str().unwrap(), + ColumnFamilyType::Lock, + key_manager.clone(), + ); + fk1.cmp(&fk2) + }); + + sw.append(&mut sd); + sw.append(&mut sl); + sw +} + impl ProxyForwarder { fn handle_ingest_sst_for_engine_store( &self, @@ -40,6 +120,7 @@ impl ProxyForwarder { )); } + let ssts_wrap = sort_sst_by_start_key(ssts_wrap, self.key_manager.clone()); for (path, cf) in &ssts_wrap { sst_views.push((path.to_str().unwrap().as_bytes(), *cf)); } diff --git a/proxy_components/engine_store_ffi/src/core/forward_raft/region.rs b/proxy_components/engine_store_ffi/src/core/forward_raft/region.rs index b8705eb3539..19c4d785a4d 100644 --- a/proxy_components/engine_store_ffi/src/core/forward_raft/region.rs +++ b/proxy_components/engine_store_ffi/src/core/forward_raft/region.rs @@ -124,8 +124,8 @@ impl ProxyForwarder { pub fn get_compact_index_and_term( &self, region_id: u64, - compact_index: u64, - compact_term: u64, + _: u64, + _: u64, ) -> Option<(u64, u64)> { let res = self .engine_store_server_helper diff --git a/proxy_components/engine_store_ffi/src/core/forwarder.rs b/proxy_components/engine_store_ffi/src/core/forwarder.rs index a301751ec9a..9f5069e05dc 100644 --- a/proxy_components/engine_store_ffi/src/core/forwarder.rs +++ b/proxy_components/engine_store_ffi/src/core/forwarder.rs @@ -1,5 +1,7 @@ // Copyright 2022 TiKV Project Authors. Licensed under Apache-2.0. +use encryption::DataKeyManager; + use crate::core::common::*; pub struct PtrWrapper(pub RawCppPtr); @@ -53,6 +55,7 @@ pub struct ProxyForwarder { pub snap_mgr: Arc, pub packed_envs: Arc, pub debug_struct: Arc, + pub key_manager: Option>, } impl ProxyForwarder { @@ -84,6 +87,7 @@ impl ProxyForwarder { snap_mgr: SnapManager, packed_envs: PackedEnvs, debug_struct: DebugStruct, + key_manager: Option>, ) -> Self { let engine_store_server_helper = gen_engine_store_server_helper(engine.proxy_ext.engine_store_server_helper); @@ -106,6 +110,7 @@ impl ProxyForwarder { snap_mgr: Arc::new(snap_mgr), packed_envs: Arc::new(packed_envs), debug_struct: Arc::new(debug_struct), + key_manager, } } @@ -130,6 +135,7 @@ impl Clone for ProxyForwarder { snap_mgr: self.snap_mgr.clone(), packed_envs: self.packed_envs.clone(), debug_struct: self.debug_struct.clone(), + key_manager: self.key_manager.clone(), } } } diff --git a/proxy_components/engine_store_ffi/src/observer.rs b/proxy_components/engine_store_ffi/src/observer.rs index 78c045d9372..008f054912d 100644 --- a/proxy_components/engine_store_ffi/src/observer.rs +++ b/proxy_components/engine_store_ffi/src/observer.rs @@ -1,6 +1,7 @@ // Copyright 2022 TiKV Project Authors. Licensed under Apache-2.0. use std::sync::Arc; +use encryption::DataKeyManager; use engine_traits::RaftEngine; use kvproto::{ raft_cmdpb::{AdminRequest, RaftCmdRequest}, @@ -42,6 +43,7 @@ impl TiFlashObserver { snap_mgr: SnapManager, packed_envs: PackedEnvs, debug_struct: DebugStruct, + key_manager: Option>, ) -> Self { TiFlashObserver { forwarder: ProxyForwarder::new( @@ -53,6 +55,7 @@ impl TiFlashObserver { snap_mgr, packed_envs, debug_struct, + key_manager, ), } } diff --git a/proxy_components/mock-engine-store/src/mock_cluster/cluster_ext.rs b/proxy_components/mock-engine-store/src/mock_cluster/cluster_ext.rs index 745ed868b59..8e176f84c89 100644 --- a/proxy_components/mock-engine-store/src/mock_cluster/cluster_ext.rs +++ b/proxy_components/mock-engine-store/src/mock_cluster/cluster_ext.rs @@ -41,6 +41,7 @@ pub struct TestData { pub expected_self_safe_ts: u64, } +#[allow(clippy::type_complexity)] #[derive(Default)] pub struct ClusterExt { // Helper to set ffi_helper_set. @@ -49,12 +50,26 @@ pub struct ClusterExt { pub(crate) ffi_helper_set: Arc>>, pub test_data: TestData, pub cluster_ptr: isize, + pub pre_run_node_callback: Option>, } impl ClusterExt { + pub fn pre_node_start(&mut self, cfg: &mut crate::mock_cluster::MixedClusterConfig) { + if let Some(c) = self.pre_run_node_callback.as_mut() { + c(cfg); + } + } + + pub fn post_cluster_start(&mut self) { + self.iter_ffi_helpers(None, &mut |_, ffi: &mut FFIHelperSet| { + ffi.proxy.refresh_cluster_raftstore_version(-1); + }); + } + pub fn get_cluster_size(&self) -> usize { self.ffi_helper_set.lock().expect("poisoned").len() } + pub fn make_ffi_helper_set_no_bind( id: u64, engines: Engines, @@ -64,6 +79,7 @@ impl ClusterExt { cluster_ptr: isize, cluster_ext_ptr: isize, mock_cfg: MockConfig, + pd_client: Option>, ) -> (FFIHelperSet, TikvConfig) { // We must allocate on heap to avoid move. let proxy = Box::new(engine_store_ffi::ffi::RaftStoreProxy::new( @@ -80,6 +96,7 @@ impl ClusterExt { }, None, None, + pd_client, )); let proxy_ref = proxy.as_ref(); @@ -132,6 +149,7 @@ impl ClusterExt { engines: Engines, key_manager: &Option>, router: &Option>, + pd_client: Option>, ) { init_global_ffi_helper_set(); // We don't know `node_id` now. @@ -145,6 +163,7 @@ impl ClusterExt { cluster_ptr, cluster_ext_ptr, mock_cfg, + pd_client, ); // We can not use moved or cloned engines any more. diff --git a/proxy_components/mock-engine-store/src/mock_cluster/v1/cluster.rs b/proxy_components/mock-engine-store/src/mock_cluster/v1/cluster.rs index c7ddcba340a..f771934d3f2 100644 --- a/proxy_components/mock-engine-store/src/mock_cluster/v1/cluster.rs +++ b/proxy_components/mock-engine-store/src/mock_cluster/v1/cluster.rs @@ -281,8 +281,9 @@ impl> Cluster { let key_manager = self.key_managers.last().unwrap().clone(); let node_id = { let mut sim = self.sim.wl(); - let cfg = self.cfg.clone(); + let mut cfg = self.cfg.clone(); // Like TiKVServer::init + self.cluster_ext.pre_node_start(&mut cfg); sim.run_node( 0, cfg, @@ -303,6 +304,7 @@ impl> Cluster { } assert_eq!(self.count, self.engines.len()); assert_eq!(self.count, self.dbs.len()); + self.cluster_ext.post_cluster_start(); Ok(()) } @@ -337,6 +339,7 @@ impl> Cluster { if let Some(labels) = self.labels.get(&node_id) { cfg.server.labels = labels.to_owned(); } + self.cluster_ext.pre_node_start(&mut cfg); let store_meta = match self.store_metas.entry(node_id) { MapEntry::Occupied(o) => { let mut meta = o.get().lock().unwrap(); diff --git a/proxy_components/mock-engine-store/src/mock_cluster/v1/node.rs b/proxy_components/mock-engine-store/src/mock_cluster/v1/node.rs index 79102eb44b7..a51115f48c0 100644 --- a/proxy_components/mock-engine-store/src/mock_cluster/v1/node.rs +++ b/proxy_components/mock-engine-store/src/mock_cluster/v1/node.rs @@ -293,6 +293,7 @@ impl Simulator for NodeCluster { None, ); + let key_mgr_cloned = key_manager.clone(); let (snap_mgr, snap_mgr_path) = if node_id == 0 || !self .trans @@ -348,6 +349,7 @@ impl Simulator for NodeCluster { snap_mgr.clone(), packed_envs, DebugStruct::default(), + key_mgr_cloned, ); tiflash_ob.register_to(&mut coprocessor_host); diff --git a/proxy_components/mock-engine-store/src/mock_cluster/v1/server.rs b/proxy_components/mock-engine-store/src/mock_cluster/v1/server.rs index 287f57bd22c..b584c7bb105 100644 --- a/proxy_components/mock-engine-store/src/mock_cluster/v1/server.rs +++ b/proxy_components/mock-engine-store/src/mock_cluster/v1/server.rs @@ -288,6 +288,7 @@ impl ServerCluster { } } + let key_mgr_cloned = key_manager.clone(); let local_reader = LocalReader::new( engines.kv.clone(), StoreMetaDelegate::new(store_meta.clone(), engines.kv.clone()), @@ -556,6 +557,7 @@ impl ServerCluster { snap_mgr.clone(), packed_envs, DebugStruct::default(), + key_mgr_cloned, ); tiflash_ob.register_to(&mut coprocessor_host); diff --git a/proxy_components/mock-engine-store/src/mock_cluster/v1/util.rs b/proxy_components/mock-engine-store/src/mock_cluster/v1/util.rs index 049eb58c438..0e7c39d0472 100644 --- a/proxy_components/mock-engine-store/src/mock_cluster/v1/util.rs +++ b/proxy_components/mock-engine-store/src/mock_cluster/v1/util.rs @@ -37,6 +37,7 @@ pub fn create_tiflash_test_engine_with_cluster_ctx>( engines, &key_manager, &router, + Some(cluster.pd_client.clone()), ); let ffi_helper_set = cluster.cluster_ext.ffi_helper_lst.last_mut().unwrap(); let engines = ffi_helper_set diff --git a/proxy_components/mock-engine-store/src/mock_store/mock_engine_store_server.rs b/proxy_components/mock-engine-store/src/mock_store/mock_engine_store_server.rs index 3342ea64853..d96e6690824 100644 --- a/proxy_components/mock-engine-store/src/mock_store/mock_engine_store_server.rs +++ b/proxy_components/mock-engine-store/src/mock_store/mock_engine_store_server.rs @@ -523,6 +523,7 @@ unsafe extern "C" fn ffi_handle_compute_store_stats( unsafe extern "C" fn ffi_get_flushed_state( _arg1: *mut interfaces_ffi::EngineStoreServerWrap, _region_id: u64, + _acquire_lock: u8 ) -> interfaces_ffi::FlushedState { interfaces_ffi::FlushedState { applied_index: 0, diff --git a/proxy_components/proxy_ffi/Cargo.toml b/proxy_components/proxy_ffi/Cargo.toml index b41d1ea6b50..9510c20ffbe 100644 --- a/proxy_components/proxy_ffi/Cargo.toml +++ b/proxy_components/proxy_ffi/Cargo.toml @@ -23,6 +23,7 @@ keys = { workspace = true, default-features = false } kvproto = { workspace = true } lazy_static = "1.3" protobuf = { version = "2.8", features = ["bytes"] } +pd_client = { workspace = true } raftstore = { workspace = true, default-features = false } slog = { version = "2.3", features = ["max_level_trace", "release_max_level_debug"] } slog-global = { version = "0.1", git = "https://github.com/breeswish/slog-global.git", rev = "d592f88e4dbba5eb439998463054f1a44fbf17b9" } @@ -30,6 +31,8 @@ tikv_util = { workspace = true, default-features = false } tokio = { version = "1.5", features = ["sync", "rt-multi-thread"] } tokio-timer = { git = "https://github.com/tikv/tokio", branch = "tokio-timer-hotfix" } tracker = { workspace = true, default-features = false } +reqwest = { version = "0.11", features = ["blocking"] } +url = "2.4.0" [dependencies.rocksdb] git = "https://github.com/tikv/rust-rocksdb.git" diff --git a/proxy_components/proxy_ffi/src/interfaces.rs b/proxy_components/proxy_ffi/src/interfaces.rs index 8b4c0fccf06..dd4b9496958 100644 --- a/proxy_components/proxy_ffi/src/interfaces.rs +++ b/proxy_components/proxy_ffi/src/interfaces.rs @@ -136,6 +136,13 @@ pub mod root { Stopping = 2, Terminated = 3, } + #[repr(u8)] + #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)] + pub enum RaftstoreVer { + Uncertain = 0, + V1 = 1, + V2 = 2, + } pub type RawCppPtrType = u32; pub type RawRustPtrType = u32; #[repr(C)] @@ -426,6 +433,13 @@ pub mod root { applied_index: u64, ), >, + pub fn_get_cluster_raftstore_version: ::std::option::Option< + unsafe extern "C" fn( + arg1: root::DB::RaftStoreProxyPtr, + refresh_strategy: u8, + timeout_ms: i64, + ) -> root::DB::RaftstoreVer, + >, } #[repr(C)] #[derive(Debug)] @@ -637,7 +651,14 @@ pub mod root { ) -> root::DB::FlushedState, >, } - pub const RAFT_STORE_PROXY_VERSION: u64 = 12138733948555436956; + extern "C" { + pub fn ffi_get_server_info_from_proxy( + arg1: isize, + arg2: root::DB::BaseBuffView, + arg3: root::DB::RawVoidPtr, + ) -> u32; + } + pub const RAFT_STORE_PROXY_VERSION: u64 = 4409497044760140371; pub const RAFT_STORE_PROXY_MAGIC_NUMBER: u32 = 324508639; } } diff --git a/proxy_components/proxy_ffi/src/raftstore_proxy.rs b/proxy_components/proxy_ffi/src/raftstore_proxy.rs index 4f1e482eed0..5eeba08f397 100644 --- a/proxy_components/proxy_ffi/src/raftstore_proxy.rs +++ b/proxy_components/proxy_ffi/src/raftstore_proxy.rs @@ -6,12 +6,15 @@ use std::sync::{ }; use encryption::DataKeyManager; +use pd_client::PdClient; +use tikv_util::error; +use tokio::runtime::Runtime; use super::{ get_engine_store_server_helper, interfaces_ffi, interfaces_ffi::{ - ConstRawVoidPtr, KVGetStatus, RaftProxyStatus, RaftStoreProxyPtr, RawCppStringPtr, - RawVoidPtr, + ConstRawVoidPtr, KVGetStatus, RaftProxyStatus, RaftStoreProxyPtr, RaftstoreVer, + RawCppStringPtr, RawVoidPtr, }, raftstore_proxy_helper_impls::*, read_index_helper, @@ -26,6 +29,8 @@ pub struct RaftStoreProxy { read_index_client: Option>, raftstore_proxy_engine: RwLock>, apply_router_client: Option>, + pd_client: Option>, + cluster_raftstore_ver: RwLock, } impl RaftStoreProxy { @@ -35,6 +40,7 @@ impl RaftStoreProxy { read_index_client: Option>, raftstore_proxy_engine: Option, apply_router_client: Option>, + pd_client: Option>, ) -> Self { RaftStoreProxy { status, @@ -42,13 +48,178 @@ impl RaftStoreProxy { read_index_client, raftstore_proxy_engine: RwLock::new(raftstore_proxy_engine), apply_router_client, + pd_client, + cluster_raftstore_ver: RwLock::new(RaftstoreVer::Uncertain), } } } +pub fn maybe_use_backup_addr(u: &str, backup: impl Fn() -> String) -> Option { + let mut res = None; + let mut need_backup_ip = false; + + if let Ok(mut stuff) = url::Url::parse(u) { + match stuff.host() { + None => { + need_backup_ip = true; + } + Some(url::Host::Domain(e)) => { + if e == "localhost" { + need_backup_ip = true; + } + } + Some(url::Host::Ipv4(e)) => { + let is_loopback_or_unspecified = e.is_unspecified() || e.is_loopback(); + if is_loopback_or_unspecified { + need_backup_ip = true; + } + } + Some(url::Host::Ipv6(e)) => { + let is_loopback_or_unspecified = e.is_unspecified() || e.is_loopback(); + if is_loopback_or_unspecified { + need_backup_ip = true; + } + } + }; + if need_backup_ip { + let mut s = backup(); + if !s.starts_with("http") { + s = format!("http://{}", s); + } + if let Ok(back) = url::Url::parse(&s) { + stuff.set_host(back.host_str()).unwrap(); + } + res = Some(stuff.to_string()) + } + } + res +} + impl RaftStoreProxy { - pub fn raftstore_version(&self) -> u64 { - 1 + pub fn cluster_raftstore_version(&self) -> RaftstoreVer { + *self.cluster_raftstore_ver.read().unwrap() + } + + /// Issue requests to all stores which is not marked as TiFlash. + /// Use the result of the first store which is not a Uncertain. + /// Or set the result to Uncertain if timeout. + pub fn refresh_cluster_raftstore_version(&mut self, timeout_ms: i64) -> bool { + let generate_request_with_timeout = |timeout_ms: i64| -> Option { + let headers = reqwest::header::HeaderMap::new(); + let mut builder = reqwest::Client::builder().default_headers(headers); + if timeout_ms >= 0 { + builder = builder.timeout(std::time::Duration::from_millis(timeout_ms as u64)); + } + match builder.build() { + Ok(o) => Some(o), + Err(e) => { + error!("generate_request_with_timeout error {:?}", e); + None + } + } + }; + + let parse_response = + |rt: &Runtime, resp: Result| -> RaftstoreVer { + match resp { + Ok(resp) => { + if resp.status() == 404 { + // If the port is not implemented. + return RaftstoreVer::V1; + } else if resp.status() != 200 { + return RaftstoreVer::Uncertain; + } + let resp = rt.block_on(async { resp.text().await }).unwrap(); + if resp.contains("partitioned") { + RaftstoreVer::V2 + } else { + RaftstoreVer::V1 + } + } + Err(e) => { + error!("get_engine_type respond error {:?}", e); + RaftstoreVer::Uncertain + } + } + }; + + // We don't use information stored in `GlobalReplicationState` to decouple. + *self.cluster_raftstore_ver.write().unwrap() = RaftstoreVer::Uncertain; + let stores = match self.pd_client.as_ref().unwrap().get_all_stores(false) { + Ok(stores) => stores, + Err(e) => { + tikv_util::info!("get_all_stores error {:?}", e); + return false; + } + }; + + let to_try_addrs = stores.iter().filter_map(|store| { + // There are some other labels such like tiflash_compute. + let shall_filter = store + .get_labels() + .iter() + .any(|label| label.get_key() == "engine" && label.get_value().contains("tiflash")); + if !shall_filter { + // TiKV's status server don't support https. + let mut u = format!("http://{}/{}", store.get_status_address(), "engine_type"); + if let Some(nu) = maybe_use_backup_addr(&u, || store.get_address().to_string()) { + tikv_util::info!("switch from {} to {}", u, nu); + u = nu; + } + // A invalid url may lead to 404, which will enforce a V1 inference, which is + // error. + if let Ok(stuff) = url::Url::parse(&u) { + if stuff.path() == "/engine_type" { + Some(u) + } else { + None + } + } else { + None + } + } else { + None + } + }); + + let rt = Runtime::new().unwrap(); + + let mut pending = vec![]; + for addr in to_try_addrs { + if let Some(c) = generate_request_with_timeout(timeout_ms) { + let _g = rt.enter(); + let f = c.get(&addr).send(); + pending.push(rt.spawn(f)); + } + } + + if pending.is_empty() { + tikv_util::error!("no valid tikv stores with status server"); + } + + loop { + if pending.is_empty() { + break; + } + let sel = futures::future::select_all(pending); + let (resp, _completed_idx, remaining) = rt.block_on(async { sel.await }); + + let res = parse_response(&rt, resp.unwrap()); + + if res != RaftstoreVer::Uncertain { + *self.cluster_raftstore_ver.write().unwrap() = res; + rt.shutdown_timeout(std::time::Duration::from_millis(1)); + return true; + } + + pending = remaining; + } + rt.shutdown_timeout(std::time::Duration::from_millis(1)); + false + } + + pub fn raftstore_version(&self) -> RaftstoreVer { + RaftstoreVer::V1 } pub fn set_kv_engine(&mut self, kv_engine: Option) { @@ -84,7 +255,7 @@ impl RaftStoreProxy { ) -> KVGetStatus { let region_state_key = keys::region_state_key(region_id); let mut res = KVGetStatus::NotFound; - if self.raftstore_version() == 1 { + if self.raftstore_version() == RaftstoreVer::V1 { self.get_value_cf(engine_traits::CF_RAFT, ®ion_state_key, &mut |value| { match value { Ok(v) => { @@ -115,7 +286,7 @@ impl RaftStoreProxy { } pub fn get_raft_apply_state(&self, _region_id: u64) -> interfaces_ffi::KVGetStatus { - if self.raftstore_version() == 1 { + if self.raftstore_version() == RaftstoreVer::V1 { panic!("wrong raftstore version"); } else { unreachable!() @@ -163,6 +334,9 @@ impl RaftStoreProxyPtr { pub unsafe fn as_ref(&self) -> &RaftStoreProxy { &*(self.inner as *const RaftStoreProxy) } + pub unsafe fn as_mut(&mut self) -> &mut RaftStoreProxy { + &mut *(self.inner as *mut RaftStoreProxy) + } pub fn is_null(&self) -> bool { self.inner.is_null() } diff --git a/proxy_components/proxy_ffi/src/raftstore_proxy_helper_impls.rs b/proxy_components/proxy_ffi/src/raftstore_proxy_helper_impls.rs index e55ed6f872f..3e9bdb4cbc0 100644 --- a/proxy_components/proxy_ffi/src/raftstore_proxy_helper_impls.rs +++ b/proxy_components/proxy_ffi/src/raftstore_proxy_helper_impls.rs @@ -22,7 +22,8 @@ use super::{ engine_store_helper_impls::*, interfaces_ffi::{ BaseBuffView, CppStrVecView, KVGetStatus, RaftProxyStatus, RaftStoreProxyFFIHelper, - RaftStoreProxyPtr, RawCppPtr, RawCppStringPtr, RawRustPtr, RawVoidPtr, SSTReaderInterfaces, + RaftStoreProxyPtr, RaftstoreVer, RawCppPtr, RawCppStringPtr, RawRustPtr, RawVoidPtr, + SSTReaderInterfaces, }, read_index_helper, snapshot_reader_impls::*, @@ -87,10 +88,34 @@ impl RaftStoreProxyFFIHelper { fn_poll_timer_task: Some(ffi_poll_timer_task), fn_get_region_local_state: Some(ffi_get_region_local_state), fn_notify_compact_log: Some(ffi_notify_compact_log), + fn_get_cluster_raftstore_version: Some(ffi_get_cluster_raftstore_version), } } } +unsafe extern "C" fn ffi_get_cluster_raftstore_version( + proxy_ptr: RaftStoreProxyPtr, + refresh_strategy: u8, + timeout_ms: i64, +) -> RaftstoreVer { + if refresh_strategy == 1 { + // Force refresh + let mut proxy_ptr = proxy_ptr; + proxy_ptr + .as_mut() + .refresh_cluster_raftstore_version(timeout_ms); + } else if refresh_strategy == 2 { + // Refresh if uncertain + if proxy_ptr.as_ref().cluster_raftstore_version() == RaftstoreVer::Uncertain { + let mut proxy_ptr = proxy_ptr; + proxy_ptr + .as_mut() + .refresh_cluster_raftstore_version(timeout_ms); + } + } + proxy_ptr.as_ref().cluster_raftstore_version() +} + unsafe extern "C" fn ffi_get_region_local_state( proxy_ptr: RaftStoreProxyPtr, region_id: u64, diff --git a/proxy_components/proxy_ffi/src/snapshot_reader_impls/sst_reader_dispatcher.rs b/proxy_components/proxy_ffi/src/snapshot_reader_impls/sst_reader_dispatcher.rs index 0fdc852ef65..88e6529c131 100644 --- a/proxy_components/proxy_ffi/src/snapshot_reader_impls/sst_reader_dispatcher.rs +++ b/proxy_components/proxy_ffi/src/snapshot_reader_impls/sst_reader_dispatcher.rs @@ -28,12 +28,12 @@ impl Clone for SSTReaderInterfaces { /// All impl of SST reader will be dispatched by this ptr. impl SSTReaderPtr { - unsafe fn as_mut_sst_lock(&mut self) -> &mut LockCFFileReader { + pub unsafe fn as_mut_sst_lock(&mut self) -> &mut LockCFFileReader { assert_eq!(self.kind, SSTFormatKind::KIND_SST); &mut *(self.inner as *mut LockCFFileReader) } - unsafe fn as_mut_sst_other(&mut self) -> &mut SSTFileReader { + pub unsafe fn as_mut_sst_other(&mut self) -> &mut SSTFileReader { assert_eq!(self.kind, SSTFormatKind::KIND_SST); &mut *(self.inner as *mut SSTFileReader) } diff --git a/proxy_components/proxy_server/src/run.rs b/proxy_components/proxy_server/src/run.rs index 3277f8eb25d..6a98934ff7c 100644 --- a/proxy_components/proxy_server/src/run.rs +++ b/proxy_components/proxy_server/src/run.rs @@ -146,6 +146,14 @@ pub fn run_impl( Some(Box::new(ProxyApplyRouterHelper::new( tikv.system.as_ref().unwrap().apply_router(), ))), + Some(tikv.pd_client.clone()), + ); + info!("start probing cluster's raftstore version"); + // We wait for a maximum of 10 seconds for every store. + proxy.refresh_cluster_raftstore_version(10 * 1000); + info!( + "cluster's raftstore version is {:?}", + proxy.cluster_raftstore_version() ); let proxy_ref = &proxy; @@ -259,6 +267,7 @@ fn run_impl_only_for_decryption( Option::None, None, None, + None, ); let proxy_ref = &proxy; @@ -1174,6 +1183,7 @@ impl TiKvServer { snap_mgr.clone(), packed_envs, DebugStruct::default(), + self.core.encryption_key_manager.clone(), ); tiflash_ob.register_to(self.coprocessor_host.as_mut().unwrap()); diff --git a/proxy_components/proxy_server/src/util.rs b/proxy_components/proxy_server/src/util.rs index c00d196696a..1a5158ae335 100644 --- a/proxy_components/proxy_server/src/util.rs +++ b/proxy_components/proxy_server/src/util.rs @@ -1,6 +1,9 @@ // Copyright 2022 TiKV Project Authors. Licensed under Apache-2.0. -use std::time::{Duration, Instant}; +use std::{ + pin::Pin, + time::{Duration, Instant}, +}; use engine_store_ffi::ffi::interfaces_ffi::{BaseBuffView, RaftStoreProxyPtr, RawVoidPtr}; use futures::{compat::Future01CompatExt, executor::block_on}; @@ -74,3 +77,29 @@ pub extern "C" fn ffi_server_info( engine_store_ffi::ffi::set_server_info_resp(&resp, res); 0 } + +#[no_mangle] +pub extern "C" fn ffi_get_server_info_from_proxy( + server_helper_ptr: isize, + view: BaseBuffView, + res: RawVoidPtr, +) -> u32 { + assert_ne!(server_helper_ptr, 0); + let mut req = ServerInfoRequest::default(); + assert_ne!(view.data, std::ptr::null()); + assert_ne!(view.len, 0); + req.merge_from_bytes(view.to_slice()).unwrap(); + + let resp = server_info_for_ffi(req); + let buff = engine_store_ffi::ffi::ProtoMsgBaseBuff::new(&resp); + unsafe { + let server_helper = &(*(server_helper_ptr + as *const engine_store_ffi::ffi::interfaces_ffi::EngineStoreServerHelper)); + server_helper.set_pb_msg_by_bytes( + engine_store_ffi::ffi::interfaces_ffi::MsgPBType::ServerInfoResponse, + res, + Pin::new(&buff).into(), + ); + } + 0 +} diff --git a/proxy_scripts/ci_check.sh b/proxy_scripts/ci_check.sh index e170d016d2a..5dba2b118e4 100755 --- a/proxy_scripts/ci_check.sh +++ b/proxy_scripts/ci_check.sh @@ -49,6 +49,7 @@ elif [[ $M == "testnew" ]]; then cargo test --package proxy_tests --test proxy shared::store cargo test --package proxy_tests --test proxy shared::region cargo test --package proxy_tests --test proxy shared::flashback + cargo test --package proxy_tests --test proxy v2_compat::cluster_raftstore_ver cargo test --package proxy_tests --test proxy v2_compat::tablet_snapshot cargo test --package proxy_tests --test proxy v2_compat::simple_write cargo test --package proxy_tests --test proxy v1_specific::region_ext diff --git a/proxy_scripts/clippy.sh b/proxy_scripts/clippy.sh old mode 100644 new mode 100755 diff --git a/proxy_tests/Cargo.toml b/proxy_tests/Cargo.toml index 6f7a270bd94..1c96815703f 100644 --- a/proxy_tests/Cargo.toml +++ b/proxy_tests/Cargo.toml @@ -112,6 +112,7 @@ engine_test = { workspace = true } engine_traits = { workspace = true } external_storage_export = { workspace = true } file_system = { workspace = true } +hex = "0.4" hyper = { version = "0.14", default-features = false, features = ["runtime"] } keys = { workspace = true } panic_hook = { workspace = true } diff --git a/proxy_tests/proxy/mod.rs b/proxy_tests/proxy/mod.rs index 2b319d5738a..bf278bed628 100644 --- a/proxy_tests/proxy/mod.rs +++ b/proxy_tests/proxy/mod.rs @@ -7,6 +7,7 @@ #![recursion_limit = "100"] #![feature(vec_into_raw_parts)] #![feature(slice_pattern)] +#![feature(async_closure)] #[macro_use] extern crate slog_global; diff --git a/proxy_tests/proxy/shared/ingest.rs b/proxy_tests/proxy/shared/ingest.rs index a259e656612..4b9781814a4 100644 --- a/proxy_tests/proxy/shared/ingest.rs +++ b/proxy_tests/proxy/shared/ingest.rs @@ -41,6 +41,55 @@ fn test_handle_ingest_sst() { cluster.shutdown(); } +#[test] +fn test_handle_multiple_ingest_sst() { + let (mut cluster, _pd_client) = new_mock_cluster(0, 1); + let _ = cluster.run(); + + let key = "k"; + cluster.must_put(key.as_bytes(), b"v"); + let region = cluster.get_region(key.as_bytes()); + + let v = make_ssts( + &cluster, + region.get_id(), + region.get_region_epoch().clone(), + (0..100).map(|i| format!("k{}", i)).collect::>(), + 5, + ); + + let mut ssts = vec![]; + for (save, m, _) in v.iter() { + ssts.push((save.clone(), proxy_ffi::name_to_cf(m.get_cf_name()))); + } + let sorted_ssts = engine_store_ffi::core::forward_raft::sort_sst_by_start_key(ssts, None); + for sl in sorted_ssts.windows(2) { + let a = &sl[0]; + let b = &sl[1]; + let fk1 = + engine_store_ffi::core::forward_raft::get_first_key(a.0.to_str().unwrap(), a.1, None); + let fk2 = + engine_store_ffi::core::forward_raft::get_first_key(b.0.to_str().unwrap(), b.1, None); + assert!(fk1 < fk2); + } + + let mut reqs = vec![]; + for (_, m, _) in v.iter() { + reqs.push(new_ingest_sst_cmd(m.clone())); + } + + let _ = cluster.request(key.as_bytes(), reqs, false, Duration::from_secs(5), true); + + check_key(&cluster, b"k66", b"2", Some(true), Some(true), None); + + for (file, _, sst_path) in v.into_iter() { + assert!(sst_path.as_path().is_file()); + assert!(!file.as_path().is_file()); + std::fs::remove_file(sst_path.as_path()).unwrap(); + } + cluster.shutdown(); +} + #[test] fn test_invalid_ingest_sst() { let (mut cluster, _pd_client) = new_mock_cluster(0, 1); diff --git a/proxy_tests/proxy/utils/mod.rs b/proxy_tests/proxy/utils/mod.rs index dbf75edfb82..0af26a60153 100644 --- a/proxy_tests/proxy/utils/mod.rs +++ b/proxy_tests/proxy/utils/mod.rs @@ -1,6 +1,7 @@ // Copyright 2022 TiKV Project Authors. Licensed under Apache-2.0. pub mod common; +pub mod sst; pub mod v1; pub mod v1_server; pub mod v2; diff --git a/proxy_tests/proxy/utils/sst.rs b/proxy_tests/proxy/utils/sst.rs new file mode 100644 index 00000000000..33bf767ac77 --- /dev/null +++ b/proxy_tests/proxy/utils/sst.rs @@ -0,0 +1,184 @@ +// Copyright 2023 TiKV Project Authors. Licensed under Apache-2.0. + +#![allow(unused)] + +use byteorder::{BigEndian, ByteOrder}; +use mock_engine_store::interfaces_ffi::BaseBuffView; +use more_asserts::assert_gt; +use proxy_ffi::{ + interfaces_ffi::{ColumnFamilyType, EngineIteratorSeekType, SSTFormatKind}, + snapshot_reader_impls::{sst_file_reader::SSTFileReader, tablet_reader::TabletReader, *}, +}; + +pub fn from_hex(key: &str) -> Result, hex::FromHexError> { + if key.starts_with("0x") || key.starts_with("0X") { + return hex::decode(&key[2..]); + } + hex::decode(key) +} + +pub unsafe fn must_get_in_tablet_snapshot(path: &str, cf: ColumnFamilyType, key: &str) { + // key is something like + // "7480000000000000FF795F720380000000FF0000026303800000FF0000017801000000FCF9DE534E2797FB83" + // . + let reader = TabletReader::ffi_get_cf_file_reader(path, cf, None); + assert_eq!(reader.kind, SSTFormatKind::KIND_TABLET); + let encoded = from_hex(key).unwrap(); + let bf = BaseBuffView { + data: encoded.as_ptr() as *const _, + len: encoded.len() as u64, + }; + + ffi_sst_reader_seek(reader.clone(), cf, EngineIteratorSeekType::Key, bf); + assert_eq!(ffi_sst_reader_remained(reader.clone(), cf), 1); + let actual = ffi_sst_reader_key(reader.clone(), cf); + assert_eq!(actual.to_slice(), encoded); +} + +pub unsafe fn must_not_get_in_tablet_snapshot(path: &str, cf: ColumnFamilyType, key: &str) { + // key is something like + // "7480000000000000FF795F720380000000FF0000026303800000FF0000017801000000FCF9DE534E2797FB83" + // . + let reader = TabletReader::ffi_get_cf_file_reader(path, cf, None); + assert_eq!(reader.kind, SSTFormatKind::KIND_TABLET); + let encoded = from_hex(key).unwrap(); + let bf = BaseBuffView { + data: encoded.as_ptr() as *const _, + len: encoded.len() as u64, + }; + + ffi_sst_reader_seek(reader.clone(), cf, EngineIteratorSeekType::Key, bf); + if ffi_sst_reader_remained(reader.clone(), cf) == 0 { + return; + } + let actual = ffi_sst_reader_key(reader.clone(), cf); + assert_ne!(actual.to_slice(), encoded); +} + +pub mod RecordFormat { + // These are functions from TiFlash. + #![allow(non_snake_case)] + + use super::*; + const ENC_GROUP_SIZE: usize = 8; + const ENC_MARKER: u8 = 0xff; + pub fn decodeInt64(p: &[u8]) -> i64 { + const SIGN_MASK: u64 = 1u64 << 63; + let y = (BigEndian::read_u64(p) ^ SIGN_MASK) as i64; + y + } + + pub fn getHandleID(p: &[u8]) -> i64 { + decodeInt64(p) + } + + pub fn getRawTiDBPK(decode: &[u8]) -> Vec { + let user_key = getUserKey(decode); + const RAW_KEY_NO_HANDLE_SIZE: usize = 1 + 8 + 2; + user_key[RAW_KEY_NO_HANDLE_SIZE..].to_vec() + } + + pub fn getUserKey(data: &[u8]) -> Vec { + removeKeyspaceID(data) + } + + pub fn removeKeyspaceID(data: &[u8]) -> Vec { + const KEYSPACE_PREFIX_LEN: usize = 4; + const TXN_MODE_PREFIX: u8 = 'x' as u8; + if data.len() < KEYSPACE_PREFIX_LEN || data[0] != TXN_MODE_PREFIX { + return data.to_vec(); + } + data[KEYSPACE_PREFIX_LEN..].to_vec() + } + + pub fn getTableId(data: &[u8]) -> i64 { + decodeInt64(&data[1..]) + } + + fn checkKeyPaddingValid(key: &[u8], ptr: usize, pad_size: u8) -> bool { + let p = unsafe { *(&key[ptr..] as *const [u8] as *const u64) } + >> ((ENC_GROUP_SIZE - pad_size as usize) * 8); + p == 0 + } + + pub fn decodeTiKVKeyFull(key: &[u8]) -> Vec { + let mut res: Vec = vec![]; + let chunk_len: usize = ENC_GROUP_SIZE + 1; + + for ptr in (0..).step_by(chunk_len) { + if ptr + chunk_len > key.len() { + panic!("Unexpexcted EOF"); + } + let marker = key[ptr + ENC_GROUP_SIZE]; + let pad_size: u8 = ENC_MARKER - marker; + if pad_size == 0 { + let mut v = key[ptr..ptr + ENC_GROUP_SIZE].to_vec(); + res.append(&mut v); + continue; + } + if pad_size as usize > ENC_GROUP_SIZE { + panic!("key padding"); + } + let a: usize = ptr + ENC_GROUP_SIZE - (pad_size as usize); + let mut v = key[ptr..a].to_vec(); + res.append(&mut v); + + if !checkKeyPaddingValid(key, ptr, pad_size) { + panic!("Key padding, wrong end") + } + return res; + } + unreachable!() + } +} + +pub fn parse_handle_id(tikv_key: &[u8]) -> i64 { + let decode = RecordFormat::decodeTiKVKeyFull(tikv_key); + let pk = RecordFormat::getRawTiDBPK(&decode); + let handle = RecordFormat::getHandleID(&pk); + handle +} + +pub fn parse_table_id(tikv_key: &[u8]) -> i64 { + let handle = RecordFormat::getTableId(&tikv_key); + handle +} + +pub unsafe fn read_sst_file(path: &str) { + let reader = SSTFileReader::ffi_get_cf_file_reader(path, None); + assert_eq!(reader.kind, SSTFormatKind::KIND_SST); + + let mut prev: Option> = None; + loop { + let r = ffi_sst_reader_remained(reader.clone(), ColumnFamilyType::Write); + if r != 1 { + let p: &[u8] = &prev.unwrap(); + tikv_util::info!( + "End table id {} key {}", + parse_table_id(p), + parse_handle_id(p) + ); + break; + } + let k = ffi_sst_reader_key(reader.clone(), ColumnFamilyType::Write); + let ks = k.to_slice(); + if let Some(p) = prev { + let ps: &[u8] = &p; + more_asserts::assert_ge!(ks, ps); + more_asserts::assert_ge!(ks[11..], ps[11..]); + let hprev = parse_handle_id(ps); + let h = parse_handle_id(ks); + assert_gt!(h, hprev); + } else { + tikv_util::info!( + "Start table id {} key {}", + parse_table_id(ks), + parse_handle_id(ks) + ); + } + prev = Some(ks.to_vec()); + // tikv_util::info!("AAA {:?} {:?} {} {}", ks, &ks[11..], ks.len(), + // BigEndian::read_u64(&ks[12..])); + ffi_sst_reader_next(reader.clone(), ColumnFamilyType::Write); + } +} diff --git a/proxy_tests/proxy/utils/v1.rs b/proxy_tests/proxy/utils/v1.rs index 91a5c5920a3..962b727bc11 100644 --- a/proxy_tests/proxy/utils/v1.rs +++ b/proxy_tests/proxy/utils/v1.rs @@ -9,6 +9,7 @@ pub use mock_engine_store::mock_cluster::v1::{ }, Cluster, Simulator, }; +use rand::seq::SliceRandom; use sst_importer::SstImporter; use test_sst_importer::gen_sst_file_with_kvs; @@ -99,6 +100,57 @@ pub fn create_tmp_importer(cfg: &MixedClusterConfig, kv_path: &str) -> (PathBuf, (dir, importer) } +pub fn make_ssts( + cluster: &Cluster, + region_id: u64, + region_epoch: RegionEpoch, + keys: Vec, + split_num: usize, +) -> Vec<(PathBuf, SstMeta, PathBuf)> { + let path = cluster.engines.iter().last().unwrap().1.kv.path(); + let (import_dir, importer) = create_tmp_importer(&cluster.cfg, path); + + // Prepare data + let mut kvs: Vec<(&[u8], &[u8])> = Vec::new(); + let mut keys = keys; + keys.sort(); + for i in 0..keys.len() { + kvs.push((keys[i].as_bytes(), b"2")); + } + + assert!(keys.len() > split_num); + let per_file_num = keys.len() / split_num; + + // Make files + let mut res = vec![]; + for i in 0..split_num { + let (import_dir, importer) = create_tmp_importer(cluster.get_config(), path); + let sst_path = import_dir.join(format!("test{}.sst", i)); + let (mut meta, data) = if i == split_num - 1 { + gen_sst_file_with_kvs(&sst_path, &kvs[i * per_file_num..]) + } else { + gen_sst_file_with_kvs(&sst_path, &kvs[i * per_file_num..(i + 1) * per_file_num]) + }; + assert!(Path::new(sst_path.to_str().unwrap()).exists()); + meta.set_region_id(region_id); + meta.set_region_epoch(region_epoch.clone()); + meta.set_cf_name("default".to_owned()); + let mut file = importer.create(&meta).unwrap(); + file.append(&data).unwrap(); + file.finish().unwrap(); + + // copy file to save dir. + let src = sst_path.clone(); + let dst = file.get_import_path().save.to_str().unwrap(); + let _ = std::fs::copy(src.clone(), dst); + res.push((file.get_import_path().save.clone(), meta, sst_path)); + } + + let mut rnd = rand::thread_rng(); + res.shuffle(&mut rnd); + res +} + pub fn make_sst( cluster: &Cluster, region_id: u64, diff --git a/proxy_tests/proxy/v2_compat/cluster_raftstore_ver.rs b/proxy_tests/proxy/v2_compat/cluster_raftstore_ver.rs new file mode 100644 index 00000000000..e6b8827c2ec --- /dev/null +++ b/proxy_tests/proxy/v2_compat/cluster_raftstore_ver.rs @@ -0,0 +1,300 @@ +// Copyright 2023 TiKV Project Authors. Licensed under Apache-2.0. + +use std::{ + convert::Infallible, + net::SocketAddr, + sync::{atomic::AtomicUsize, Arc, RwLock}, +}; + +use hyper::{ + service::{make_service_fn, service_fn}, + Body, Response, Server, +}; +use proxy_ffi::interfaces_ffi::RaftstoreVer; +use tokio::{runtime::Runtime, sync::oneshot, task::JoinHandle}; + +use crate::utils::v1::*; + +type BoxError = Box; + +#[derive(Clone, Copy, Debug, PartialEq)] +enum ReturnState { + V1, + V2, + C404, + C403, + TimeoutV1, +} + +#[derive(Clone)] +pub struct SharedState { + inner: Arc>, +} + +impl SharedState { + fn new(t: ReturnState) -> Self { + Self { + inner: Arc::new(RwLock::new(t)), + } + } +} + +async fn handle_request(shared: SharedState) -> Result, BoxError> { + let x = shared.inner.read().unwrap().clone(); + match x { + ReturnState::C403 => Ok(Response::builder() + .status(403) + .body("raft-kv".into()) + .unwrap()), + ReturnState::C404 => Ok(Response::builder() + .status(404) + .body("raft-kv".into()) + .unwrap()), + ReturnState::V1 => Ok(Response::new("raft-kv".into())), + ReturnState::V2 => Ok(Response::new("partitioned-raft-kv".into())), + ReturnState::TimeoutV1 => { + tokio::time::sleep(std::time::Duration::from_millis(1000)).await; + Ok(Response::new("raft-kv".into())) + } + } +} + +pub async fn serve(rx: tokio::sync::oneshot::Receiver, port: u16, state: SharedState) { + let addr = SocketAddr::from(([127, 0, 0, 1], port)); + + let state = state.clone(); + let make_svc = make_service_fn(move |_| { + let state = state.clone(); + async move { + // This is the request handler. + Ok::<_, Infallible>(service_fn(move |_| { + let state = state.clone(); + handle_request(state) + })) + } + }); + + let server = Server::bind(&addr).serve(make_svc); + + let sig = async move || { + rx.await.unwrap(); + }; + let graceful = server.with_graceful_shutdown(sig()); + + if let Err(e) = graceful.await { + eprintln!("server error: {}", e); + } +} + +struct MockServer { + rt: Runtime, + tx: Vec>, + res: Vec, + handle: Vec>, + index: Arc, + ports: Vec, +} + +impl MockServer { + fn new(ports: Vec) -> Self { + let rt = Runtime::new().unwrap(); + let n = ports.len(); + let mut tx = vec![]; + let mut res = vec![]; + let mut handle = vec![]; + let index = Arc::new(AtomicUsize::new(0)); + for i in 0..n { + let (tx1, rx1) = oneshot::channel(); + let res1 = SharedState::new(ReturnState::V2); + let f1 = rt.spawn(serve(rx1, ports[i], res1.clone())); + tx.push(tx1); + res.push(res1); + handle.push(f1); + } + Self { + rt, + tx, + res, + handle, + index, + ports, + } + } +} + +impl Drop for MockServer { + fn drop(&mut self) { + let tx = std::mem::take(&mut self.tx); + let handle = std::mem::take(&mut self.handle); + for x in tx.into_iter() { + x.send(1).unwrap() + } + for x in handle.into_iter() { + self.rt.block_on(async { x }); + } + } +} + +#[test] +fn test_maybe_substitute_addr() { + let b = || "http://111.111.111.111:100".to_string(); + let b2 = || "111.111.111.111:100".to_string(); + let b3 = || "tikv2:100".to_string(); + fn test_with_b(b: impl Fn() -> String + Clone, ba: &str) { + let o = proxy_ffi::maybe_use_backup_addr( + "http://tc-tikv-9.tc-tikv-peer.mutiple-rocksdb-btdpb.svc:20160/engine_type", + b.clone(), + ); + assert_eq!(o, None); + let o = proxy_ffi::maybe_use_backup_addr("http://1.2.3.4:5/engine_type", b.clone()); + assert_eq!(o, None); + let o = proxy_ffi::maybe_use_backup_addr("http://127.0.0.1/engine_type", b.clone()); + assert_eq!(o.unwrap(), format!("http://{}/engine_type", ba)); + let o = proxy_ffi::maybe_use_backup_addr("http://localhost:222/a", b.clone()); + assert_eq!(o.unwrap(), format!("http://{}:222/a", ba)); + let o = proxy_ffi::maybe_use_backup_addr("http://0.0.0.0:333/engine_type", b.clone()); + assert_eq!(o.unwrap(), format!("http://{}:333/engine_type", ba)); + } + test_with_b(b, "111.111.111.111"); + test_with_b(b2, "111.111.111.111"); + test_with_b(b3, "tikv2"); +} + +#[test] +fn test_with_error_status_addr() { + let mock_server = MockServer::new(vec![1111, 1112]); + let (mut cluster_v1, _) = new_mock_cluster(1, 3); + let index = mock_server.index.clone(); + cluster_v1.cluster_ext.pre_run_node_callback = + Some(Box::new(move |cfg: &mut MixedClusterConfig| { + cfg.server.labels.clear(); + match index.load(Ordering::Relaxed) { + 0 => cfg.server.status_addr = "error$#_string".to_string(), + 1 => cfg.server.status_addr = "".to_string(), + _ => cfg.server.status_addr = "localhost:1119".to_string(), + } + index.fetch_add(1, Ordering::Relaxed); + })); + cluster_v1.run(); + cluster_v1 + .cluster_ext + .iter_ffi_helpers(None, &mut |_, ffi: &mut FFIHelperSet| { + ffi.proxy.refresh_cluster_raftstore_version(-1); + assert_eq!( + ffi.proxy.cluster_raftstore_version(), + RaftstoreVer::Uncertain + ); + }); + cluster_v1.shutdown(); +} + +#[test] +fn test_with_tiflash() { + let mock_server = MockServer::new(vec![1111, 1112]); + let (mut cluster_v1, _) = new_mock_cluster(1, 2); + let addrs = mock_server.ports.iter().map(|e| format!("127.0.0.1:{}", e)); + let status_addrs = Arc::new(addrs.collect::>()); + let index = mock_server.index.clone(); + cluster_v1.cluster_ext.pre_run_node_callback = + Some(Box::new(move |cfg: &mut MixedClusterConfig| { + if index.load(Ordering::Relaxed) == 0 { + cfg.server.labels.clear(); + } else { + cfg.server + .labels + .insert("engine".to_string(), "tiflash".to_string()); + } + cfg.server.status_addr = (*status_addrs)[index.load(Ordering::Relaxed)].to_string(); + index.fetch_add(1, Ordering::Relaxed); + })); + cluster_v1.run(); + + // TiFlash will always output as V1, however, we should neglect that. + + *mock_server.res[0].inner.write().unwrap() = ReturnState::C403; + // Node 1 is TiFlash node, its result will be neglected. + *mock_server.res[1].inner.write().unwrap() = ReturnState::V2; + cluster_v1 + .cluster_ext + .iter_ffi_helpers(None, &mut |_, ffi: &mut FFIHelperSet| { + ffi.proxy.refresh_cluster_raftstore_version(-1); + assert_eq!( + ffi.proxy.cluster_raftstore_version(), + RaftstoreVer::Uncertain + ); + }); + + cluster_v1.shutdown(); +} + +#[test] +fn test_normal() { + let mock_server = MockServer::new(vec![1111, 1112]); + let (mut cluster_v1, _) = new_mock_cluster(1, 2); + // Will switch from the ipv6 localhost address into store.get_addr(). + let addrs = mock_server.ports.iter().map(|e| format!("[::]:{}", e)); + let status_addrs = Arc::new(addrs.collect::>()); + let index = mock_server.index.clone(); + cluster_v1.cluster_ext.pre_run_node_callback = + Some(Box::new(move |cfg: &mut MixedClusterConfig| { + cfg.server.labels.clear(); + cfg.server.status_addr = (*status_addrs)[index.load(Ordering::Relaxed)].to_string(); + index.fetch_add(1, Ordering::Relaxed); + })); + cluster_v1.run(); + + cluster_v1 + .cluster_ext + .iter_ffi_helpers(None, &mut |_, ffi: &mut FFIHelperSet| { + assert_eq!(ffi.proxy.cluster_raftstore_version(), RaftstoreVer::V2); + }); + + *mock_server.res[0].inner.write().unwrap() = ReturnState::C403; + *mock_server.res[1].inner.write().unwrap() = ReturnState::V1; + cluster_v1 + .cluster_ext + .iter_ffi_helpers(None, &mut |_, ffi: &mut FFIHelperSet| { + ffi.proxy.refresh_cluster_raftstore_version(-1); + assert_eq!(ffi.proxy.cluster_raftstore_version(), RaftstoreVer::V1); + }); + + *mock_server.res[0].inner.write().unwrap() = ReturnState::TimeoutV1; + *mock_server.res[1].inner.write().unwrap() = ReturnState::V2; + assert_eq!( + *mock_server.res[0].inner.read().unwrap(), + ReturnState::TimeoutV1 + ); + assert_eq!(*mock_server.res[1].inner.read().unwrap(), ReturnState::V2); + cluster_v1 + .cluster_ext + .iter_ffi_helpers(None, &mut |_, ffi: &mut FFIHelperSet| { + ffi.proxy.refresh_cluster_raftstore_version(500); + assert_eq!(ffi.proxy.cluster_raftstore_version(), RaftstoreVer::V2); + }); + + // All timeout result in uncertain state. + *mock_server.res[0].inner.write().unwrap() = ReturnState::TimeoutV1; + *mock_server.res[1].inner.write().unwrap() = ReturnState::TimeoutV1; + cluster_v1 + .cluster_ext + .iter_ffi_helpers(None, &mut |_, ffi: &mut FFIHelperSet| { + ffi.proxy.refresh_cluster_raftstore_version(500); + assert_eq!( + ffi.proxy.cluster_raftstore_version(), + RaftstoreVer::Uncertain + ); + }); + + // If returns 404, means the server is an old v1 TiKV which doesn't have this + // service. + *mock_server.res[0].inner.write().unwrap() = ReturnState::C404; + *mock_server.res[1].inner.write().unwrap() = ReturnState::C404; + cluster_v1 + .cluster_ext + .iter_ffi_helpers(None, &mut |_, ffi: &mut FFIHelperSet| { + ffi.proxy.refresh_cluster_raftstore_version(-1); + assert_eq!(ffi.proxy.cluster_raftstore_version(), RaftstoreVer::V1); + }); + + cluster_v1.shutdown(); +} diff --git a/proxy_tests/proxy/v2_compat/mod.rs b/proxy_tests/proxy/v2_compat/mod.rs index f641166686f..9e5a468e48c 100644 --- a/proxy_tests/proxy/v2_compat/mod.rs +++ b/proxy_tests/proxy/v2_compat/mod.rs @@ -1,5 +1,6 @@ // Copyright 2023 TiKV Project Authors. Licensed under Apache-2.0. +mod cluster_raftstore_ver; mod simple_write; mod tablet_snapshot; pub(crate) mod utils; diff --git a/proxy_tests/proxy/v2_compat/tablet_snapshot.rs b/proxy_tests/proxy/v2_compat/tablet_snapshot.rs index ecc3f907c79..c58219911fe 100644 --- a/proxy_tests/proxy/v2_compat/tablet_snapshot.rs +++ b/proxy_tests/proxy/v2_compat/tablet_snapshot.rs @@ -21,7 +21,7 @@ use tikv::server::tablet_snap::send_snap as send_snap_v2; use tikv_util::time::Limiter; use super::utils::*; -use crate::utils::v1::*; +use crate::utils::{sst::*, v1::*}; fn random_long_vec(length: usize) -> Vec { let mut rng = rand::thread_rng(); diff --git a/raftstore-proxy/ffi/src/RaftStoreProxyFFI/@version b/raftstore-proxy/ffi/src/RaftStoreProxyFFI/@version index 536dcf0d27b..391f11db197 100644 --- a/raftstore-proxy/ffi/src/RaftStoreProxyFFI/@version +++ b/raftstore-proxy/ffi/src/RaftStoreProxyFFI/@version @@ -1,3 +1,3 @@ #pragma once #include -namespace DB { constexpr uint64_t RAFT_STORE_PROXY_VERSION = 12138733948555436956ull; } \ No newline at end of file +namespace DB { constexpr uint64_t RAFT_STORE_PROXY_VERSION = 4409497044760140371ull; } \ No newline at end of file diff --git a/raftstore-proxy/ffi/src/RaftStoreProxyFFI/ProxyFFI.h b/raftstore-proxy/ffi/src/RaftStoreProxyFFI/ProxyFFI.h index 8bf9d3d0394..4caf9c25713 100644 --- a/raftstore-proxy/ffi/src/RaftStoreProxyFFI/ProxyFFI.h +++ b/raftstore-proxy/ffi/src/RaftStoreProxyFFI/ProxyFFI.h @@ -85,6 +85,12 @@ enum class EngineStoreServerStatus : uint8_t { // shutdown. }; +enum class RaftstoreVer : uint8_t { + Uncertain = 0, + V1, + V2, +}; + using RawCppPtrType = uint32_t; using RawRustPtrType = uint32_t; @@ -243,6 +249,9 @@ struct RaftStoreProxyFFIHelper { void (*fn_notify_compact_log)(RaftStoreProxyPtr, uint64_t region_id, uint64_t compact_index, uint64_t compact_term, uint64_t applied_index); + RaftstoreVer (*fn_get_cluster_raftstore_version)(RaftStoreProxyPtr, + uint8_t refresh_strategy, + int64_t timeout_ms); }; struct PageStorageInterfaces { @@ -316,4 +325,15 @@ struct EngineStoreServerHelper { uint64_t region_id, uint8_t acquire_lock); }; + +#ifdef __cplusplus +extern "C" { +#endif +// Basically same as ffi_server_info, but no need to setup ProxyHelper, only +// need to setup ServerHelper. Used when proxy not start. +uint32_t ffi_get_server_info_from_proxy(intptr_t, BaseBuffView, RawVoidPtr); +#ifdef __cplusplus +} +#endif + } // namespace DB From 90e8a744b5b6c1399c60a5238310523c57672cf0 Mon Sep 17 00:00:00 2001 From: CalvinNeo Date: Wed, 12 Jul 2023 14:16:27 +0800 Subject: [PATCH 05/12] exec_compact_log not regard to flushed state Signed-off-by: CalvinNeo --- components/raftstore/src/coprocessor/mod.rs | 7 +-- components/raftstore/src/store/fsm/apply.rs | 46 +++++++++------ .../src/core/forward_raft/region.rs | 7 +-- .../mock_store/mock_engine_store_server.rs | 2 +- proxy_tests/proxy/shared/write.rs | 56 +++++++++++++++++++ 5 files changed, 88 insertions(+), 30 deletions(-) diff --git a/components/raftstore/src/coprocessor/mod.rs b/components/raftstore/src/coprocessor/mod.rs index f50d0c85c92..f7568f76e57 100644 --- a/components/raftstore/src/coprocessor/mod.rs +++ b/components/raftstore/src/coprocessor/mod.rs @@ -352,12 +352,7 @@ pub trait RegionChangeObserver: Coprocessor { } /// Returns the maximum index the underlying engine can compact. - fn get_compact_index_and_term( - &self, - _: u64, - _: u64, - _: u64, - ) -> Option<(u64, u64)> { + fn get_compact_index_and_term(&self, _: u64, _: u64, _: u64) -> Option<(u64, u64)> { None } } diff --git a/components/raftstore/src/store/fsm/apply.rs b/components/raftstore/src/store/fsm/apply.rs index b8884c682c9..e2918437f88 100644 --- a/components/raftstore/src/store/fsm/apply.rs +++ b/components/raftstore/src/store/fsm/apply.rs @@ -3057,7 +3057,7 @@ where &mut self, voter_replicated_index: u64, voter_replicated_term: u64, - handle_queue: bool, + compact_log_in_queue: bool, ) -> Result<(bool, Option>)> { PEER_ADMIN_CMD_COUNTER.compact.all.inc(); let first_index = entry_storage::first_index(&self.apply_state); @@ -3072,7 +3072,7 @@ where return Ok((false, None)); } - if !handle_queue { + if !compact_log_in_queue { // When `compact_log_in_queue` is set to false. // It it a compact request from underlying engine. let mut compact_index = voter_replicated_index; @@ -3245,21 +3245,32 @@ where } } - if let Some((custom_compact_index, custom_compact_term)) = ctx - .host - .get_compact_index_and_term(self.region_id(), compact_index, compact_term) - { - if custom_compact_index < compact_index { - // If the underlying engine has not compact to expected yet, - // We will compact to what the underlying engine tells which is a best effort. - compact_index = custom_compact_index; - compact_term = custom_compact_term; - } - if compact_index > self.max_compact_index { - self.max_compact_index = compact_index; - self.max_compact_term = compact_term; - } + // if let Some((custom_compact_index, custom_compact_term)) = ctx + // .host + // .get_compact_index_and_term(self.region_id(), compact_index, + // compact_term) { + // if custom_compact_index < compact_index { + // // This may happen when the underlying engine has done a flush before + // this CompactLog, but to a smaller index. // Previously, if + // the underlying engine has not compact to expected yet, // we + // will just reject the CompactLog by `pre_exec`. // Now, We + // will compact to where the underlying engine tells which is a best effort. + // compact_index = custom_compact_index; + // compact_term = custom_compact_term; + // } + // if compact_index > self.max_compact_index { + // self.max_compact_index = compact_index; + // self.max_compact_term = compact_term; + // } + // } + + // Safety: compact index is monotonicly increased guarded by `compact_raft_log` + // and `entry_storage::first_index`. + if compact_index > self.max_compact_index { + self.max_compact_index = compact_index; + self.max_compact_term = compact_term; } + if compact_index < first_index { debug!( "compact index < first index, no need to compact"; @@ -4396,10 +4407,11 @@ where return; } + let compact_log_in_queue = ctx.host.compact_log_in_queue(); let res = self.delegate.try_compact_log( voter_replicated_index, voter_replicated_term, - ctx.host.compact_log_in_queue(), + compact_log_in_queue, ); match res { Ok((should_write, res)) => { diff --git a/proxy_components/engine_store_ffi/src/core/forward_raft/region.rs b/proxy_components/engine_store_ffi/src/core/forward_raft/region.rs index 19c4d785a4d..674bfa0c8ca 100644 --- a/proxy_components/engine_store_ffi/src/core/forward_raft/region.rs +++ b/proxy_components/engine_store_ffi/src/core/forward_raft/region.rs @@ -121,12 +121,7 @@ impl ProxyForwarder { .unwrap(); } - pub fn get_compact_index_and_term( - &self, - region_id: u64, - _: u64, - _: u64, - ) -> Option<(u64, u64)> { + pub fn get_compact_index_and_term(&self, region_id: u64, _: u64, _: u64) -> Option<(u64, u64)> { let res = self .engine_store_server_helper .get_persisted_state(region_id); diff --git a/proxy_components/mock-engine-store/src/mock_store/mock_engine_store_server.rs b/proxy_components/mock-engine-store/src/mock_store/mock_engine_store_server.rs index d96e6690824..958ae7f4332 100644 --- a/proxy_components/mock-engine-store/src/mock_store/mock_engine_store_server.rs +++ b/proxy_components/mock-engine-store/src/mock_store/mock_engine_store_server.rs @@ -523,7 +523,7 @@ unsafe extern "C" fn ffi_handle_compute_store_stats( unsafe extern "C" fn ffi_get_flushed_state( _arg1: *mut interfaces_ffi::EngineStoreServerWrap, _region_id: u64, - _acquire_lock: u8 + _acquire_lock: u8, ) -> interfaces_ffi::FlushedState { interfaces_ffi::FlushedState { applied_index: 0, diff --git a/proxy_tests/proxy/shared/write.rs b/proxy_tests/proxy/shared/write.rs index a5432fae073..e29589460b2 100644 --- a/proxy_tests/proxy/shared/write.rs +++ b/proxy_tests/proxy/shared/write.rs @@ -365,6 +365,62 @@ fn test_unsupport_admin_cmd() { cluster.shutdown(); } +#[test] +fn test_failed_compact_log() { + let (mut cluster, _pd_client) = new_mock_cluster(0, 3); + disable_auto_gen_compact_log(&mut cluster); + cluster.run(); + + cluster.must_put(b"k", b"v"); + let region = cluster.get_region("k".as_bytes()); + let region_id = region.get_id(); + fail::cfg("try_flush_data", "return(1)").unwrap(); + for i in 0..5 { + let k = format!("k{}", i); + let v = format!("v{}", i); + cluster.must_put(k.as_bytes(), v.as_bytes()); + } + for i in 0..5 { + let k = format!("k{}", i); + let v = format!("v{}", i); + check_key(&cluster, k.as_bytes(), v.as_bytes(), Some(true), None, None); + } + std::thread::sleep(std::time::Duration::from_millis(500)); + let prev_state = collect_all_states(&cluster.cluster_ext, region_id); + let (compact_index, compact_term) = get_valid_compact_index(&prev_state); + let compact_log = test_raftstore::new_compact_log_request(compact_index, compact_term); + let req = test_raftstore::new_admin_request(region_id, region.get_region_epoch(), compact_log); + let res = cluster + .call_command_on_leader(req, Duration::from_secs(3)) + .unwrap(); + // compact index should less than applied index + assert!(!res.get_header().has_error(), "{:?}", res); + + for i in 5..10 { + let k = format!("k{}", i); + let v = format!("v{}", i); + cluster.must_put(k.as_bytes(), v.as_bytes()); + } + for i in 5..10 { + let k = format!("k{}", i); + let v = format!("v{}", i); + check_key(&cluster, k.as_bytes(), v.as_bytes(), Some(true), None, None); + } + std::thread::sleep(std::time::Duration::from_millis(500)); + let prev_state = collect_all_states(&cluster.cluster_ext, region_id); + let compact_log = test_raftstore::new_compact_log_request(1, compact_term); + let req = test_raftstore::new_admin_request(region_id, region.get_region_epoch(), compact_log); + let res = cluster + .call_command_on_leader(req, Duration::from_secs(3)) + .unwrap(); + // compact index should less than applied index + assert!(!res.get_header().has_error(), "{:?}", res); + + let new_state = collect_all_states(&cluster.cluster_ext, region_id); + must_unaltered_disk_truncated_state(&prev_state, &new_state); + cluster.shutdown(); +} + #[test] fn test_compact_log() { let (mut cluster, _pd_client) = new_mock_cluster(0, 3); From 8183ecc8eae955d785678bbd135e67b9d9fa9717 Mon Sep 17 00:00:00 2001 From: test Date: Mon, 17 Jul 2023 13:48:30 +0800 Subject: [PATCH 06/12] add tests Signed-off-by: test --- components/raftstore/src/store/fsm/apply.rs | 2 +- .../mock-engine-store/src/mock_cluster/v1/cluster.rs | 7 +++++-- .../src/mock_cluster/v1/cluster_ext_v1.rs | 4 +++- proxy_components/proxy_ffi/src/raftstore_proxy.rs | 10 ++++++++++ .../proxy_ffi/src/raftstore_proxy_helper_impls.rs | 1 + proxy_scripts/ci_check.sh | 1 + proxy_tests/proxy/shared/mod.rs | 1 + 7 files changed, 22 insertions(+), 4 deletions(-) diff --git a/components/raftstore/src/store/fsm/apply.rs b/components/raftstore/src/store/fsm/apply.rs index e2918437f88..ab27ce10cde 100644 --- a/components/raftstore/src/store/fsm/apply.rs +++ b/components/raftstore/src/store/fsm/apply.rs @@ -3163,7 +3163,7 @@ where fn exec_compact_log( &mut self, - ctx: &mut ApplyContext, + ctx: &ApplyContext, req: &AdminRequest, ) -> Result<(AdminResponse, ApplyResult)> { PEER_ADMIN_CMD_COUNTER.compact.all.inc(); diff --git a/proxy_components/mock-engine-store/src/mock_cluster/v1/cluster.rs b/proxy_components/mock-engine-store/src/mock_cluster/v1/cluster.rs index f771934d3f2..5f018c3375c 100644 --- a/proxy_components/mock-engine-store/src/mock_cluster/v1/cluster.rs +++ b/proxy_components/mock-engine-store/src/mock_cluster/v1/cluster.rs @@ -259,7 +259,6 @@ impl> Cluster { debug!("recover node"; "node_id" => node_id); // Like TiKVServer::init self.run_node(node_id)?; - self.post_node_start(node_id); } // Try start new nodes. @@ -271,6 +270,7 @@ impl> Cluster { } let (router, system) = create_raft_batch_system(&self.cfg.raft_store, &self.resource_manager); + let apply_router = system.apply_router(); self.create_engine(Some(router.clone())); let store_meta = Arc::new(Mutex::new(StoreMeta::new(PENDING_MSG_CAP))); @@ -300,7 +300,7 @@ impl> Cluster { self.store_metas.insert(node_id, store_meta); self.key_managers_map.insert(node_id, key_manager.clone()); self.register_ffi_helper_set(None, node_id); - self.post_node_start(node_id); + self.post_node_start(node_id, apply_router); } assert_eq!(self.count, self.engines.len()); assert_eq!(self.count, self.dbs.len()); @@ -355,10 +355,13 @@ impl> Cluster { tikv_util::thread_group::set_properties(Some(props)); debug!("calling run node"; "node_id" => node_id); + let apply_router = system.apply_router(); // FIXME: rocksdb event listeners may not work, because we change the router. self.sim .wl() .run_node(node_id, cfg, engines, store_meta, key_mgr, router, system)?; + + self.post_node_start(node_id, apply_router); debug!("node {} started", node_id); Ok(()) } diff --git a/proxy_components/mock-engine-store/src/mock_cluster/v1/cluster_ext_v1.rs b/proxy_components/mock-engine-store/src/mock_cluster/v1/cluster_ext_v1.rs index 8d8cef7491e..a869a643af3 100644 --- a/proxy_components/mock-engine-store/src/mock_cluster/v1/cluster_ext_v1.rs +++ b/proxy_components/mock-engine-store/src/mock_cluster/v1/cluster_ext_v1.rs @@ -6,6 +6,7 @@ use collections::HashMap; use engine_store_ffi::{ffi::RaftStoreProxyFFI, TiFlashEngine}; use engine_tiflash::DB; use engine_traits::{Engines, KvEngine}; +use raftstore::store::fsm::ApplyRouter; use tikv_util::{sys::SysQuota, HandyRwLock}; use super::{common::*, Cluster, Simulator}; @@ -15,7 +16,7 @@ impl> Cluster { self.cluster_ext.access_ffi_helpers(f) } - pub fn post_node_start(&mut self, node_id: u64) { + pub fn post_node_start(&mut self, node_id: u64, apply_router: ApplyRouter) { // Since we use None to create_ffi_helper_set, we must init again. let router = self.sim.rl().get_router(node_id).unwrap(); self.cluster_ext @@ -26,6 +27,7 @@ impl> Cluster { SysQuota::cpu_cores_quota() as usize * 2, ), ))); + ffi.proxy.setup_apply_router_helper(apply_router.clone()); }); } diff --git a/proxy_components/proxy_ffi/src/raftstore_proxy.rs b/proxy_components/proxy_ffi/src/raftstore_proxy.rs index 5eeba08f397..699ede40cba 100644 --- a/proxy_components/proxy_ffi/src/raftstore_proxy.rs +++ b/proxy_components/proxy_ffi/src/raftstore_proxy.rs @@ -6,7 +6,9 @@ use std::sync::{ }; use encryption::DataKeyManager; +use engine_traits::KvEngine; use pd_client::PdClient; +use raftstore::store::fsm::ApplyRouter; use tikv_util::error; use tokio::runtime::Runtime; @@ -330,6 +332,14 @@ impl RaftStoreProxyFFI for RaftStoreProxy { } } +impl RaftStoreProxy { + pub fn setup_apply_router_helper(&mut self, ar: ApplyRouter) { + self.apply_router_client = Some(Box::new( + crate::apply_router_helper::ProxyApplyRouterHelper::new(ar), + )); + } +} + impl RaftStoreProxyPtr { pub unsafe fn as_ref(&self) -> &RaftStoreProxy { &*(self.inner as *const RaftStoreProxy) diff --git a/proxy_components/proxy_ffi/src/raftstore_proxy_helper_impls.rs b/proxy_components/proxy_ffi/src/raftstore_proxy_helper_impls.rs index 3e9bdb4cbc0..1d53771fc46 100644 --- a/proxy_components/proxy_ffi/src/raftstore_proxy_helper_impls.rs +++ b/proxy_components/proxy_ffi/src/raftstore_proxy_helper_impls.rs @@ -299,6 +299,7 @@ pub extern "C" fn ffi_notify_compact_log( assert!(!proxy_ptr.is_null()); unsafe { if proxy_ptr.as_ref().maybe_apply_router_helper().is_none() { + tikv_util::info!("Apply router helper is none"); return; } } diff --git a/proxy_scripts/ci_check.sh b/proxy_scripts/ci_check.sh index 5dba2b118e4..bf7391d9bb5 100755 --- a/proxy_scripts/ci_check.sh +++ b/proxy_scripts/ci_check.sh @@ -38,6 +38,7 @@ elif [[ $M == "testnew" ]]; then cargo check --package proxy_server --features="$ENABLE_FEATURES" # tests based on mock-engine-store, with compat for new proxy cargo test --package proxy_tests --test proxy shared::write + cargo test --package proxy_tests --test proxy shared::proactive_flush cargo test --package proxy_tests --test proxy shared::snapshot cargo test --package proxy_tests --test proxy shared::normal::store cargo test --package proxy_tests --test proxy shared::normal::config diff --git a/proxy_tests/proxy/shared/mod.rs b/proxy_tests/proxy/shared/mod.rs index 12e24c39e70..9070b6989a2 100644 --- a/proxy_tests/proxy/shared/mod.rs +++ b/proxy_tests/proxy/shared/mod.rs @@ -7,6 +7,7 @@ mod ffi; mod ingest; mod mock; mod normal; +mod proactive_flush; mod region; mod replica_read; mod server_cluster_test; From 714b54f3c5d8fd227852645b702c1dfce971d3ac Mon Sep 17 00:00:00 2001 From: CalvinNeo Date: Tue, 18 Jul 2023 13:48:41 +0800 Subject: [PATCH 07/12] remove get_compact_index_and_term Signed-off-by: CalvinNeo --- .../raftstore/src/coprocessor/dispatcher.rs | 16 ----- components/raftstore/src/coprocessor/mod.rs | 5 -- components/raftstore/src/store/fsm/apply.rs | 65 +++++++++---------- components/raftstore/src/store/fsm/peer.rs | 2 +- .../src/core/forward_raft/region.rs | 10 --- .../engine_store_ffi/src/observer.rs | 10 --- .../proxy_ffi/src/apply_router_helper.rs | 2 +- 7 files changed, 31 insertions(+), 79 deletions(-) diff --git a/components/raftstore/src/coprocessor/dispatcher.rs b/components/raftstore/src/coprocessor/dispatcher.rs index d0e374c48ea..faca9e5d089 100644 --- a/components/raftstore/src/coprocessor/dispatcher.rs +++ b/components/raftstore/src/coprocessor/dispatcher.rs @@ -861,22 +861,6 @@ impl CoprocessorHost { return true; } - pub fn get_compact_index_and_term( - &self, - region_id: u64, - compact_index: u64, - compact_term: u64, - ) -> Option<(u64, u64)> { - for observer in &self.registry.region_change_observers { - let observer = observer.observer.inner(); - let res = observer.get_compact_index_and_term(region_id, compact_index, compact_term); - if res.is_some() { - return res; - } - } - return None; - } - pub fn shutdown(&self) { for entry in &self.registry.admin_observers { entry.observer.inner().stop(); diff --git a/components/raftstore/src/coprocessor/mod.rs b/components/raftstore/src/coprocessor/mod.rs index f7568f76e57..787be23df32 100644 --- a/components/raftstore/src/coprocessor/mod.rs +++ b/components/raftstore/src/coprocessor/mod.rs @@ -350,11 +350,6 @@ pub trait RegionChangeObserver: Coprocessor { fn compact_log_in_queue(&self) -> bool { true } - - /// Returns the maximum index the underlying engine can compact. - fn get_compact_index_and_term(&self, _: u64, _: u64, _: u64) -> Option<(u64, u64)> { - None - } } pub trait MessageObserver: Coprocessor { diff --git a/components/raftstore/src/store/fsm/apply.rs b/components/raftstore/src/store/fsm/apply.rs index ab27ce10cde..8535800464b 100644 --- a/components/raftstore/src/store/fsm/apply.rs +++ b/components/raftstore/src/store/fsm/apply.rs @@ -1198,6 +1198,24 @@ where }); } + fn write_apply_state_by_underlying_engine( + &self, + wb: &mut EK::WriteBatch, + apply_state: &RaftApplyState, + ) { + wb.put_msg_cf( + CF_RAFT, + &keys::apply_state_key(self.region.get_id()), + apply_state, + ) + .unwrap_or_else(|e| { + panic!( + "{} failed to save apply state to write batch, error: {:?}", + self.tag, e + ); + }); + } + fn maybe_write_apply_state(&self, apply_ctx: &mut ApplyContext) { let can_write = apply_ctx.host.pre_write_apply_state(&self.region); if can_write { @@ -3245,25 +3263,6 @@ where } } - // if let Some((custom_compact_index, custom_compact_term)) = ctx - // .host - // .get_compact_index_and_term(self.region_id(), compact_index, - // compact_term) { - // if custom_compact_index < compact_index { - // // This may happen when the underlying engine has done a flush before - // this CompactLog, but to a smaller index. // Previously, if - // the underlying engine has not compact to expected yet, // we - // will just reject the CompactLog by `pre_exec`. // Now, We - // will compact to where the underlying engine tells which is a best effort. - // compact_index = custom_compact_index; - // compact_term = custom_compact_term; - // } - // if compact_index > self.max_compact_index { - // self.max_compact_index = compact_index; - // self.max_compact_term = compact_term; - // } - // } - // Safety: compact index is monotonicly increased guarded by `compact_raft_log` // and `entry_storage::first_index`. if compact_index > self.max_compact_index { @@ -3852,7 +3851,7 @@ where region_id: u64, voter_replicated_index: u64, voter_replicated_term: u64, - applied_index: u64, + applied_index: Option, }, } @@ -3938,7 +3937,7 @@ where } => { write!( f, - "[region {}] check compact, voter_replicated_index: {}, voter_replicated_term: {}, applied_index: {}", + "[region {}] check compact, voter_replicated_index: {}, voter_replicated_term: {}, applied_index: {:?}", region_id, voter_replicated_index, voter_replicated_term, applied_index ) } @@ -4401,7 +4400,7 @@ where ctx: &mut ApplyContext, voter_replicated_index: u64, voter_replicated_term: u64, - applied_index: u64, + maybe_applied_index: Option, ) { if self.delegate.pending_remove || self.delegate.stopped { return; @@ -4419,29 +4418,23 @@ where if ctx.timer.is_none() { ctx.timer = Some(Instant::now_coarse()); } - let origin_applied_index = self.delegate.apply_state.get_applied_index(); - // flush with applied_index that TiFlash has flushed cache. - self.delegate.apply_state.set_applied_index(applied_index); ctx.prepare_for(&mut self.delegate); let mut result = VecDeque::new(); // If modified `truncated_state` in `try_compact_log`, the apply state should be // persisted. if should_write { - self.delegate.write_apply_state(ctx.kv_wb_mut()); + if let Some(underlying_engine_applied) = maybe_applied_index.as_ref() { + let mut state = self.delegate.apply_state.clone(); + state.set_applied_index(*underlying_engine_applied); + self.delegate + .write_apply_state_by_underlying_engine(ctx.kv_wb_mut(), &state); + } else { + self.delegate.write_apply_state(ctx.kv_wb_mut()); + }; ctx.commit_opt(&mut self.delegate, true); } result.push_back(res); - self.delegate - .apply_state - .set_applied_index(origin_applied_index); ctx.finish_for(&mut self.delegate, result); - - info!( - "trigger compact log"; - "region_id" => self.delegate.region.get_id(), - "applied_index" => applied_index, - "origin applied_index" => origin_applied_index, - ); } } Err(e) => error!(?e; diff --git a/components/raftstore/src/store/fsm/peer.rs b/components/raftstore/src/store/fsm/peer.rs index 60c1e7ad11a..d658f969d45 100644 --- a/components/raftstore/src/store/fsm/peer.rs +++ b/components/raftstore/src/store/fsm/peer.rs @@ -2736,7 +2736,7 @@ where region_id: self.region_id(), voter_replicated_index, voter_replicated_term, - applied_index: self.fsm.peer.get_store().applied_index(), + applied_index: None, }, ) } diff --git a/proxy_components/engine_store_ffi/src/core/forward_raft/region.rs b/proxy_components/engine_store_ffi/src/core/forward_raft/region.rs index 674bfa0c8ca..b402124d54b 100644 --- a/proxy_components/engine_store_ffi/src/core/forward_raft/region.rs +++ b/proxy_components/engine_store_ffi/src/core/forward_raft/region.rs @@ -120,14 +120,4 @@ impl ProxyForwarder { .access_cached_region_info_mut(region_id, f) .unwrap(); } - - pub fn get_compact_index_and_term(&self, region_id: u64, _: u64, _: u64) -> Option<(u64, u64)> { - let res = self - .engine_store_server_helper - .get_persisted_state(region_id); - if res.0 == 0 && res.1 == 0 { - return None; - } - Some(res) - } } diff --git a/proxy_components/engine_store_ffi/src/observer.rs b/proxy_components/engine_store_ffi/src/observer.rs index 008f054912d..f3f2972c2ff 100644 --- a/proxy_components/engine_store_ffi/src/observer.rs +++ b/proxy_components/engine_store_ffi/src/observer.rs @@ -195,16 +195,6 @@ impl RegionChangeObserver for TiFlashObs fn compact_log_in_queue(&self) -> bool { false } - - fn get_compact_index_and_term( - &self, - region_id: u64, - compact_index: u64, - compact_term: u64, - ) -> Option<(u64, u64)> { - self.forwarder - .get_compact_index_and_term(region_id, compact_index, compact_term) - } } impl MessageObserver for TiFlashObserver { diff --git a/proxy_components/proxy_ffi/src/apply_router_helper.rs b/proxy_components/proxy_ffi/src/apply_router_helper.rs index bc2eb846566..dc5d8e67e5d 100644 --- a/proxy_components/proxy_ffi/src/apply_router_helper.rs +++ b/proxy_components/proxy_ffi/src/apply_router_helper.rs @@ -39,7 +39,7 @@ impl ApplyRouterHelper for ProxyApplyRouterHelper { region_id, voter_replicated_index: compact_index, voter_replicated_term: compact_term, - applied_index, + applied_index: Some(applied_index), }, ) } From 56a5ece6d04d9d9de58598a739b321c24a2b7868 Mon Sep 17 00:00:00 2001 From: CalvinNeo Date: Tue, 18 Jul 2023 15:59:45 +0800 Subject: [PATCH 08/12] add observer Signed-off-by: CalvinNeo --- .../raftstore/src/coprocessor/dispatcher.rs | 22 +++++++++++-- components/raftstore/src/coprocessor/mod.rs | 13 ++++++-- components/raftstore/src/store/fsm/apply.rs | 31 ++++++++++++++----- components/raftstore/src/store/fsm/peer.rs | 1 + .../src/core/forward_raft/region.rs | 17 ++++++++++ .../engine_store_ffi/src/core/forwarder.rs | 1 + .../engine_store_ffi/src/observer.rs | 21 +++++++++++-- proxy_components/engine_tiflash/src/engine.rs | 1 + .../src/proxy_utils/proxy_ext.rs | 14 ++++++++- .../proxy_ffi/src/apply_router_helper.rs | 1 + 10 files changed, 105 insertions(+), 17 deletions(-) diff --git a/components/raftstore/src/coprocessor/dispatcher.rs b/components/raftstore/src/coprocessor/dispatcher.rs index faca9e5d089..4c53d3cbe32 100644 --- a/components/raftstore/src/coprocessor/dispatcher.rs +++ b/components/raftstore/src/coprocessor/dispatcher.rs @@ -853,12 +853,28 @@ impl CoprocessorHost { } } - pub fn compact_log_in_queue(&self) -> bool { + pub fn post_compact_log_from_underlying_engine( + &self, + do_write: bool, + compact_index: u64, + compact_term: u64, + max_compact_index: u64, + max_compact_term: u64, + request_applied_index: u64, + raftstore_applied_index: u64, + ) { for observer in &self.registry.region_change_observers { let observer = observer.observer.inner(); - return observer.compact_log_in_queue(); + return observer.post_compact_log_from_underlying_engine( + do_write, + compact_index, + compact_term, + max_compact_index, + max_compact_term, + request_applied_index, + raftstore_applied_index, + ); } - return true; } pub fn shutdown(&self) { diff --git a/components/raftstore/src/coprocessor/mod.rs b/components/raftstore/src/coprocessor/mod.rs index 787be23df32..126301428f6 100644 --- a/components/raftstore/src/coprocessor/mod.rs +++ b/components/raftstore/src/coprocessor/mod.rs @@ -346,9 +346,16 @@ pub trait RegionChangeObserver: Coprocessor { true } - /// Returns true if need to get compact log index from pending queue. - fn compact_log_in_queue(&self) -> bool { - true + fn post_compact_log_from_underlying_engine( + &self, + do_write: bool, + compact_index: u64, + compact_term: u64, + max_compact_index: u64, + max_compact_term: u64, + request_applied_index: u64, + raftstore_applied_index: u64, + ) { } } diff --git a/components/raftstore/src/store/fsm/apply.rs b/components/raftstore/src/store/fsm/apply.rs index 8535800464b..e184b3c352c 100644 --- a/components/raftstore/src/store/fsm/apply.rs +++ b/components/raftstore/src/store/fsm/apply.rs @@ -3075,7 +3075,7 @@ where &mut self, voter_replicated_index: u64, voter_replicated_term: u64, - compact_log_in_queue: bool, + from_underlying_engine: bool, ) -> Result<(bool, Option>)> { PEER_ADMIN_CMD_COUNTER.compact.all.inc(); let first_index = entry_storage::first_index(&self.apply_state); @@ -3090,9 +3090,7 @@ where return Ok((false, None)); } - if !compact_log_in_queue { - // When `compact_log_in_queue` is set to false. - // It it a compact request from underlying engine. + if from_underlying_engine { let mut compact_index = voter_replicated_index; // `compact_index` is reported by underlying engine, it may be greater than the // recorded `max_compact_index`. We will use the smaller one of this @@ -3852,6 +3850,7 @@ where voter_replicated_index: u64, voter_replicated_term: u64, applied_index: Option, + from_underlying_engine: bool, }, } @@ -3934,11 +3933,16 @@ where voter_replicated_index, voter_replicated_term, applied_index, + from_underlying_engine, } => { write!( f, - "[region {}] check compact, voter_replicated_index: {}, voter_replicated_term: {}, applied_index: {:?}", - region_id, voter_replicated_index, voter_replicated_term, applied_index + "[region {}] check compact, voter_replicated_index: {}, voter_replicated_term: {}, applied_index: {:?}, from_underlying_engine {}", + region_id, + voter_replicated_index, + voter_replicated_term, + applied_index, + from_underlying_engine ) } } @@ -4401,16 +4405,16 @@ where voter_replicated_index: u64, voter_replicated_term: u64, maybe_applied_index: Option, + from_underlying_engine: bool, ) { if self.delegate.pending_remove || self.delegate.stopped { return; } - let compact_log_in_queue = ctx.host.compact_log_in_queue(); let res = self.delegate.try_compact_log( voter_replicated_index, voter_replicated_term, - compact_log_in_queue, + from_underlying_engine, ); match res { Ok((should_write, res)) => { @@ -4422,6 +4426,15 @@ where let mut result = VecDeque::new(); // If modified `truncated_state` in `try_compact_log`, the apply state should be // persisted. + ctx.host.post_compact_log_from_underlying_engine( + should_write, + voter_replicated_index, + voter_replicated_term, + self.delegate.max_compact_index, + self.delegate.max_compact_term, + maybe_applied_index.unwrap_or(0), + self.delegate.apply_state.get_applied_index(), + ); if should_write { if let Some(underlying_engine_applied) = maybe_applied_index.as_ref() { let mut state = self.delegate.apply_state.clone(); @@ -4524,6 +4537,7 @@ where voter_replicated_index, voter_replicated_term, applied_index, + from_underlying_engine, .. } => { self.check_pending_compact_log( @@ -4531,6 +4545,7 @@ where voter_replicated_index, voter_replicated_term, applied_index, + from_underlying_engine, ); } } diff --git a/components/raftstore/src/store/fsm/peer.rs b/components/raftstore/src/store/fsm/peer.rs index d658f969d45..43ab9ffdd0d 100644 --- a/components/raftstore/src/store/fsm/peer.rs +++ b/components/raftstore/src/store/fsm/peer.rs @@ -2737,6 +2737,7 @@ where voter_replicated_index, voter_replicated_term, applied_index: None, + from_underlying_engine: false, }, ) } diff --git a/proxy_components/engine_store_ffi/src/core/forward_raft/region.rs b/proxy_components/engine_store_ffi/src/core/forward_raft/region.rs index b402124d54b..f4eff617714 100644 --- a/proxy_components/engine_store_ffi/src/core/forward_raft/region.rs +++ b/proxy_components/engine_store_ffi/src/core/forward_raft/region.rs @@ -120,4 +120,21 @@ impl ProxyForwarder { .access_cached_region_info_mut(region_id, f) .unwrap(); } + + pub fn post_compact_log_from_underlying_engine( + &self, + _: bool, + _: u64, + _: u64, + _: u64, + _: u64, + _: u64, + _: u64, + ) { + self.engine + .proxy_ext + .debug_struct + .proactive_compact_log_count + .fetch_add(1, Ordering::SeqCst); + } } diff --git a/proxy_components/engine_store_ffi/src/core/forwarder.rs b/proxy_components/engine_store_ffi/src/core/forwarder.rs index 9f5069e05dc..8f3c9f12304 100644 --- a/proxy_components/engine_store_ffi/src/core/forwarder.rs +++ b/proxy_components/engine_store_ffi/src/core/forwarder.rs @@ -1,6 +1,7 @@ // Copyright 2022 TiKV Project Authors. Licensed under Apache-2.0. use encryption::DataKeyManager; +use portable_atomic::AtomicUsize; use crate::core::common::*; diff --git a/proxy_components/engine_store_ffi/src/observer.rs b/proxy_components/engine_store_ffi/src/observer.rs index f3f2972c2ff..753ef3941bc 100644 --- a/proxy_components/engine_store_ffi/src/observer.rs +++ b/proxy_components/engine_store_ffi/src/observer.rs @@ -192,8 +192,25 @@ impl RegionChangeObserver for TiFlashObs self.forwarder.pre_write_apply_state(ob_ctx.region()) } - fn compact_log_in_queue(&self) -> bool { - false + fn post_compact_log_from_underlying_engine( + &self, + do_write: bool, + compact_index: u64, + compact_term: u64, + max_compact_index: u64, + max_compact_term: u64, + request_applied_index: u64, + raftstore_applied_index: u64, + ) { + self.forwarder.post_compact_log_from_underlying_engine( + do_write, + compact_index, + compact_term, + max_compact_index, + max_compact_term, + request_applied_index, + raftstore_applied_index, + ) } } diff --git a/proxy_components/engine_tiflash/src/engine.rs b/proxy_components/engine_tiflash/src/engine.rs index 488ffab81bb..3cf2f84ebc6 100644 --- a/proxy_components/engine_tiflash/src/engine.rs +++ b/proxy_components/engine_tiflash/src/engine.rs @@ -66,6 +66,7 @@ impl RocksEngine { engine_store_hub, config_set, cached_region_info_manager: Some(Arc::new(crate::CachedRegionInfoManager::new())), + debug_struct: Arc::new(Default::default()), }; let ps_ext = PageStorageExt { engine_store_server_helper, diff --git a/proxy_components/engine_tiflash/src/proxy_utils/proxy_ext.rs b/proxy_components/engine_tiflash/src/proxy_utils/proxy_ext.rs index a03cfbd623b..6f85fc09ece 100644 --- a/proxy_components/engine_tiflash/src/proxy_utils/proxy_ext.rs +++ b/proxy_components/engine_tiflash/src/proxy_utils/proxy_ext.rs @@ -2,13 +2,23 @@ use std::{ fmt::Formatter, sync::{ - atomic::{AtomicIsize, Ordering}, + atomic::{AtomicIsize, AtomicU64, Ordering}, Arc, }, }; use crate::proxy_utils::EngineStoreHub; +#[cfg(not(feature = "testexport"))] +#[derive(Debug, Default)] +pub struct DebugStruct {} + +#[cfg(feature = "testexport")] +#[derive(Debug, Default)] +pub struct DebugStruct { + pub proactive_compact_log_count: AtomicU64, +} + // This struct should be safe to copy. #[derive(Clone)] pub struct ProxyEngineExt { @@ -18,6 +28,7 @@ pub struct ProxyEngineExt { pub engine_store_hub: Option>, pub config_set: Option>, pub cached_region_info_manager: Option>, + pub debug_struct: Arc, } impl std::fmt::Debug for ProxyEngineExt { @@ -45,6 +56,7 @@ impl Default for ProxyEngineExt { engine_store_hub: None, config_set: None, cached_region_info_manager: None, + debug_struct: Arc::new(Default::default()), } } } diff --git a/proxy_components/proxy_ffi/src/apply_router_helper.rs b/proxy_components/proxy_ffi/src/apply_router_helper.rs index dc5d8e67e5d..5dd20a81af8 100644 --- a/proxy_components/proxy_ffi/src/apply_router_helper.rs +++ b/proxy_components/proxy_ffi/src/apply_router_helper.rs @@ -40,6 +40,7 @@ impl ApplyRouterHelper for ProxyApplyRouterHelper { voter_replicated_index: compact_index, voter_replicated_term: compact_term, applied_index: Some(applied_index), + from_underlying_engine: true, }, ) } From c6aa3728744654d5708c916707cb05273928fd16 Mon Sep 17 00:00:00 2001 From: CalvinNeo Date: Tue, 18 Jul 2023 16:56:44 +0800 Subject: [PATCH 09/12] add test Signed-off-by: CalvinNeo --- proxy_tests/proxy/shared/proactive_flush.rs | 81 +++++++++++++++++++++ 1 file changed, 81 insertions(+) create mode 100644 proxy_tests/proxy/shared/proactive_flush.rs diff --git a/proxy_tests/proxy/shared/proactive_flush.rs b/proxy_tests/proxy/shared/proactive_flush.rs new file mode 100644 index 00000000000..0cfe9be78b1 --- /dev/null +++ b/proxy_tests/proxy/shared/proactive_flush.rs @@ -0,0 +1,81 @@ +// Copyright 2023 TiKV Project Authors. Licensed under Apache-2.0. +use proxy_ffi::interfaces_ffi::{RaftStoreProxyFFIHelper, RaftStoreProxyPtr}; + +use crate::utils::v1::*; + +fn do_notify_compact_log( + helper: &RaftStoreProxyFFIHelper, + ptr: RaftStoreProxyPtr, + region_id: u64, + compact_index: u64, + compact_term: u64, + applied_index: u64, +) { + unsafe { + (helper.fn_notify_compact_log.as_ref().unwrap())( + ptr, + region_id, + compact_index, + compact_term, + applied_index, + ) + } +} + +#[test] +fn test_proactive_flush() { + let (mut cluster, _) = new_mock_cluster(0, 1); + disable_auto_gen_compact_log(&mut cluster); + fail::cfg("try_flush_data", "return(0)").unwrap(); + cluster.run(); + + let region = cluster.get_region(b"k1"); + let region_id = region.get_id(); + must_put_and_check_key(&mut cluster, 1, 10, Some(true), None, None); + + let prev_state = collect_all_states(&cluster.cluster_ext, region_id); + let (compact_index, compact_term) = get_valid_compact_index(&prev_state); + let compact_log = test_raftstore::new_compact_log_request(compact_index, compact_term); + let req = test_raftstore::new_admin_request(region_id, region.get_region_epoch(), compact_log); + let _ = cluster + .call_command_on_leader(req, Duration::from_secs(3)) + .unwrap(); + + std::thread::sleep(std::time::Duration::from_secs(1)); + + let new_state = collect_all_states(&cluster.cluster_ext, region_id); + must_unaltered_disk_apply_state(&prev_state, &new_state); + let prev_state = new_state; + + let applied_index = compact_index; + iter_ffi_helpers(&cluster, Some(vec![1]), &mut |_, ffi: &mut FFIHelperSet| { + do_notify_compact_log( + ffi.proxy_helper.as_ref(), + ffi.proxy_helper.proxy_ptr.clone(), + region_id, + compact_index, + compact_term, + applied_index, + ) + }); + // Wait the async scheduled + std::thread::sleep(std::time::Duration::from_secs(2)); + let new_state = collect_all_states(&cluster.cluster_ext, region_id); + must_altered_disk_apply_state(&prev_state, &new_state); + iter_ffi_helpers(&cluster, Some(vec![1]), &mut |_, ffi: &mut FFIHelperSet| { + assert_eq!( + ffi.engine_store_server + .engines + .as_ref() + .unwrap() + .kv + .proxy_ext + .debug_struct + .proactive_compact_log_count + .load(Ordering::SeqCst), + 1 + ) + }); + fail::remove("try_flush_data"); + cluster.shutdown(); +} From 76e7cd4cc8209e25672642fcda26638b6a70a058 Mon Sep 17 00:00:00 2001 From: CalvinNeo Date: Tue, 25 Jul 2023 14:14:25 +0800 Subject: [PATCH 10/12] fix region_id Signed-off-by: CalvinNeo --- .../raftstore/src/coprocessor/dispatcher.rs | 2 ++ components/raftstore/src/coprocessor/mod.rs | 15 ++++++----- components/raftstore/src/store/fsm/apply.rs | 1 + .../src/core/forward_raft/region.rs | 27 ++++++++++++++----- .../engine_store_ffi/src/observer.rs | 2 ++ 5 files changed, 33 insertions(+), 14 deletions(-) diff --git a/components/raftstore/src/coprocessor/dispatcher.rs b/components/raftstore/src/coprocessor/dispatcher.rs index 4c53d3cbe32..53188be2429 100644 --- a/components/raftstore/src/coprocessor/dispatcher.rs +++ b/components/raftstore/src/coprocessor/dispatcher.rs @@ -855,6 +855,7 @@ impl CoprocessorHost { pub fn post_compact_log_from_underlying_engine( &self, + region_id: u64, do_write: bool, compact_index: u64, compact_term: u64, @@ -866,6 +867,7 @@ impl CoprocessorHost { for observer in &self.registry.region_change_observers { let observer = observer.observer.inner(); return observer.post_compact_log_from_underlying_engine( + region_id, do_write, compact_index, compact_term, diff --git a/components/raftstore/src/coprocessor/mod.rs b/components/raftstore/src/coprocessor/mod.rs index 126301428f6..c2fd88391c5 100644 --- a/components/raftstore/src/coprocessor/mod.rs +++ b/components/raftstore/src/coprocessor/mod.rs @@ -348,13 +348,14 @@ pub trait RegionChangeObserver: Coprocessor { fn post_compact_log_from_underlying_engine( &self, - do_write: bool, - compact_index: u64, - compact_term: u64, - max_compact_index: u64, - max_compact_term: u64, - request_applied_index: u64, - raftstore_applied_index: u64, + _region_id: u64, + _do_write: bool, + _compact_index: u64, + _compact_term: u64, + _max_compact_index: u64, + _max_compact_term: u64, + _request_applied_index: u64, + _raftstore_applied_index: u64, ) { } } diff --git a/components/raftstore/src/store/fsm/apply.rs b/components/raftstore/src/store/fsm/apply.rs index e184b3c352c..2e06a33a632 100644 --- a/components/raftstore/src/store/fsm/apply.rs +++ b/components/raftstore/src/store/fsm/apply.rs @@ -4427,6 +4427,7 @@ where // If modified `truncated_state` in `try_compact_log`, the apply state should be // persisted. ctx.host.post_compact_log_from_underlying_engine( + self.delegate.region.get_id(), should_write, voter_replicated_index, voter_replicated_term, diff --git a/proxy_components/engine_store_ffi/src/core/forward_raft/region.rs b/proxy_components/engine_store_ffi/src/core/forward_raft/region.rs index f4eff617714..67e2ef08959 100644 --- a/proxy_components/engine_store_ffi/src/core/forward_raft/region.rs +++ b/proxy_components/engine_store_ffi/src/core/forward_raft/region.rs @@ -123,14 +123,27 @@ impl ProxyForwarder { pub fn post_compact_log_from_underlying_engine( &self, - _: bool, - _: u64, - _: u64, - _: u64, - _: u64, - _: u64, - _: u64, + region_id: u64, + do_write: bool, + compact_index: u64, + compact_term: u64, + max_compact_index: u64, + max_compact_term: u64, + request_applied_index: u64, + raftstore_applied_index: u64, ) { + debug!( + "post_compact_log_from_underlying_engine"; + "region_id" => region_id, + "do_write" => do_write, + "compact_index" => compact_index, + "compact_term" => compact_term, + "max_compact_index" => max_compact_index, + "max_compact_term" => max_compact_term, + "request_applied_index" => request_applied_index, + "raftstore_applied_index" => raftstore_applied_index, + ); + #[cfg(any(test, feature = "testexport"))] self.engine .proxy_ext .debug_struct diff --git a/proxy_components/engine_store_ffi/src/observer.rs b/proxy_components/engine_store_ffi/src/observer.rs index 753ef3941bc..b12b9d574f5 100644 --- a/proxy_components/engine_store_ffi/src/observer.rs +++ b/proxy_components/engine_store_ffi/src/observer.rs @@ -194,6 +194,7 @@ impl RegionChangeObserver for TiFlashObs fn post_compact_log_from_underlying_engine( &self, + region_id: u64, do_write: bool, compact_index: u64, compact_term: u64, @@ -203,6 +204,7 @@ impl RegionChangeObserver for TiFlashObs raftstore_applied_index: u64, ) { self.forwarder.post_compact_log_from_underlying_engine( + region_id, do_write, compact_index, compact_term, From fcddf52fe5f9b51f2d3b242383177b462419f265 Mon Sep 17 00:00:00 2001 From: CalvinNeo Date: Tue, 25 Jul 2023 14:33:43 +0800 Subject: [PATCH 11/12] fix clippy Signed-off-by: CalvinNeo --- components/raftstore/src/coprocessor/dispatcher.rs | 2 +- components/raftstore/src/store/fsm/apply.rs | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/components/raftstore/src/coprocessor/dispatcher.rs b/components/raftstore/src/coprocessor/dispatcher.rs index 53188be2429..27fa7b508d4 100644 --- a/components/raftstore/src/coprocessor/dispatcher.rs +++ b/components/raftstore/src/coprocessor/dispatcher.rs @@ -866,7 +866,7 @@ impl CoprocessorHost { ) { for observer in &self.registry.region_change_observers { let observer = observer.observer.inner(); - return observer.post_compact_log_from_underlying_engine( + observer.post_compact_log_from_underlying_engine( region_id, do_write, compact_index, diff --git a/components/raftstore/src/store/fsm/apply.rs b/components/raftstore/src/store/fsm/apply.rs index 2e06a33a632..c5b09484b6d 100644 --- a/components/raftstore/src/store/fsm/apply.rs +++ b/components/raftstore/src/store/fsm/apply.rs @@ -1748,7 +1748,7 @@ where AdminCmdType::ChangePeerV2 => self.exec_change_peer_v2(ctx, request), AdminCmdType::Split => self.exec_split(ctx, request), AdminCmdType::BatchSplit => self.exec_batch_split(ctx, request), - AdminCmdType::CompactLog => self.exec_compact_log(ctx, request), + AdminCmdType::CompactLog => self.exec_compact_log(request), AdminCmdType::TransferLeader => self.exec_transfer_leader(request, ctx.exec_log_term), AdminCmdType::ComputeHash => self.exec_compute_hash(ctx, request), AdminCmdType::VerifyHash => self.exec_verify_hash(ctx, request), @@ -3179,7 +3179,6 @@ where fn exec_compact_log( &mut self, - ctx: &ApplyContext, req: &AdminRequest, ) -> Result<(AdminResponse, ApplyResult)> { PEER_ADMIN_CMD_COUNTER.compact.all.inc(); From 04a72f1227ad382703d050e44c704ef54746d42f Mon Sep 17 00:00:00 2001 From: CalvinNeo Date: Tue, 25 Jul 2023 16:03:07 +0800 Subject: [PATCH 12/12] remove get_flushed_state Signed-off-by: CalvinNeo --- proxy_components/engine_store_ffi/src/core/forwarder.rs | 1 - .../src/mock_store/mock_engine_store_server.rs | 1 - .../proxy_ffi/src/engine_store_helper_impls.rs | 8 -------- proxy_components/proxy_ffi/src/interfaces.rs | 9 +-------- raftstore-proxy/ffi/src/RaftStoreProxyFFI/@version | 2 +- raftstore-proxy/ffi/src/RaftStoreProxyFFI/ProxyFFI.h | 3 --- 6 files changed, 2 insertions(+), 22 deletions(-) diff --git a/proxy_components/engine_store_ffi/src/core/forwarder.rs b/proxy_components/engine_store_ffi/src/core/forwarder.rs index 8f3c9f12304..9f5069e05dc 100644 --- a/proxy_components/engine_store_ffi/src/core/forwarder.rs +++ b/proxy_components/engine_store_ffi/src/core/forwarder.rs @@ -1,7 +1,6 @@ // Copyright 2022 TiKV Project Authors. Licensed under Apache-2.0. use encryption::DataKeyManager; -use portable_atomic::AtomicUsize; use crate::core::common::*; diff --git a/proxy_components/mock-engine-store/src/mock_store/mock_engine_store_server.rs b/proxy_components/mock-engine-store/src/mock_store/mock_engine_store_server.rs index 958ae7f4332..ba23f193ea7 100644 --- a/proxy_components/mock-engine-store/src/mock_store/mock_engine_store_server.rs +++ b/proxy_components/mock-engine-store/src/mock_store/mock_engine_store_server.rs @@ -318,7 +318,6 @@ pub fn gen_engine_store_server_helper( fn_is_ps_empty: Some(ffi_mockps_is_ps_empty), fn_handle_purge_ps: Some(ffi_mockps_handle_purge_ps), }, - fn_get_flushed_state: Some(ffi_get_flushed_state), } } diff --git a/proxy_components/proxy_ffi/src/engine_store_helper_impls.rs b/proxy_components/proxy_ffi/src/engine_store_helper_impls.rs index 686a1f75688..6e3ee116b5d 100644 --- a/proxy_components/proxy_ffi/src/engine_store_helper_impls.rs +++ b/proxy_components/proxy_ffi/src/engine_store_helper_impls.rs @@ -333,14 +333,6 @@ impl EngineStoreServerHelper { debug_assert!(self.fn_fast_add_peer.is_some()); unsafe { (self.fn_fast_add_peer.into_inner())(self.inner, region_id, new_peer_id) } } - - pub fn get_persisted_state(&self, region_id: u64) -> (u64, u64) { - debug_assert!(self.fn_get_flushed_state.is_some()); - unsafe { - let flushed_state = (self.fn_get_flushed_state.into_inner())(self.inner, region_id, 1); - return (flushed_state.applied_index, flushed_state.applied_term); - } - } } // PageStorage specific diff --git a/proxy_components/proxy_ffi/src/interfaces.rs b/proxy_components/proxy_ffi/src/interfaces.rs index dd4b9496958..2289dfb885c 100644 --- a/proxy_components/proxy_ffi/src/interfaces.rs +++ b/proxy_components/proxy_ffi/src/interfaces.rs @@ -643,13 +643,6 @@ pub mod root { new_peer_id: u64, ) -> root::DB::FastAddPeerRes, >, - pub fn_get_flushed_state: ::std::option::Option< - unsafe extern "C" fn( - arg1: *mut root::DB::EngineStoreServerWrap, - region_id: u64, - acquire_lock: u8, - ) -> root::DB::FlushedState, - >, } extern "C" { pub fn ffi_get_server_info_from_proxy( @@ -658,7 +651,7 @@ pub mod root { arg3: root::DB::RawVoidPtr, ) -> u32; } - pub const RAFT_STORE_PROXY_VERSION: u64 = 4409497044760140371; + pub const RAFT_STORE_PROXY_VERSION: u64 = 1785250247080530932; pub const RAFT_STORE_PROXY_MAGIC_NUMBER: u32 = 324508639; } } diff --git a/raftstore-proxy/ffi/src/RaftStoreProxyFFI/@version b/raftstore-proxy/ffi/src/RaftStoreProxyFFI/@version index 391f11db197..f7a1258050d 100644 --- a/raftstore-proxy/ffi/src/RaftStoreProxyFFI/@version +++ b/raftstore-proxy/ffi/src/RaftStoreProxyFFI/@version @@ -1,3 +1,3 @@ #pragma once #include -namespace DB { constexpr uint64_t RAFT_STORE_PROXY_VERSION = 4409497044760140371ull; } \ No newline at end of file +namespace DB { constexpr uint64_t RAFT_STORE_PROXY_VERSION = 1785250247080530932ull; } \ No newline at end of file diff --git a/raftstore-proxy/ffi/src/RaftStoreProxyFFI/ProxyFFI.h b/raftstore-proxy/ffi/src/RaftStoreProxyFFI/ProxyFFI.h index 4caf9c25713..1eb0dbd9de5 100644 --- a/raftstore-proxy/ffi/src/RaftStoreProxyFFI/ProxyFFI.h +++ b/raftstore-proxy/ffi/src/RaftStoreProxyFFI/ProxyFFI.h @@ -321,9 +321,6 @@ struct EngineStoreServerHelper { uint64_t leader_safe_ts); FastAddPeerRes (*fn_fast_add_peer)(EngineStoreServerWrap *, uint64_t region_id, uint64_t new_peer_id); - FlushedState (*fn_get_flushed_state)(EngineStoreServerWrap *, - uint64_t region_id, - uint8_t acquire_lock); }; #ifdef __cplusplus