From af146524fef395bf5618f565d592c68d0a076eac Mon Sep 17 00:00:00 2001 From: Yang Zhang Date: Sun, 2 Jun 2024 01:09:18 -0700 Subject: [PATCH 1/3] Upgrade rocksdb Signed-off-by: Yang Zhang --- .gitmodules | 8 +- librocksdb_sys/build.rs | 2 +- librocksdb_sys/crocksdb/c.cc | 136 +------ librocksdb_sys/crocksdb/crocksdb/c.h | 40 +- librocksdb_sys/libtitan_sys/titan | 2 +- librocksdb_sys/rocksdb | 2 +- librocksdb_sys/src/generate.py | 10 +- librocksdb_sys/src/generated.rs | 552 +++++++++++++++------------ librocksdb_sys/src/lib.rs | 30 +- src/compaction_filter.rs | 21 +- src/perf_context.rs | 19 + src/rocksdb.rs | 16 +- src/rocksdb_options.rs | 18 - tests/cases/test_rocksdb_options.rs | 5 +- 14 files changed, 369 insertions(+), 492 deletions(-) diff --git a/.gitmodules b/.gitmodules index 1dce46cb8..39b057be9 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,9 +1,9 @@ [submodule "rocksdb"] path = librocksdb_sys/rocksdb - url = https://github.com/tikv/rocksdb.git - branch = 6.29.tikv + url = https://github.com/v01dstar/rocksdb.git + branch = 8.10-tikv [submodule "titan"] path = librocksdb_sys/libtitan_sys/titan - url = https://github.com/tikv/titan.git - branch = master + url = https://github.com/v01dstar/titan.git + branch = upgrade-rocksdb diff --git a/librocksdb_sys/build.rs b/librocksdb_sys/build.rs index 06d14b68c..3d0235cff 100644 --- a/librocksdb_sys/build.rs +++ b/librocksdb_sys/build.rs @@ -76,7 +76,7 @@ fn main() { build.cpp(true).file("crocksdb/c.cc"); if env::var("CARGO_CFG_TARGET_OS").unwrap() != "windows" { - build.flag("-std=c++11"); + build.flag("-std=c++17"); build.flag("-fno-rtti"); } link_cpp(&mut build); diff --git a/librocksdb_sys/crocksdb/c.cc b/librocksdb_sys/crocksdb/c.cc index ea58d546f..df49844a4 100644 --- a/librocksdb_sys/crocksdb/c.cc +++ b/librocksdb_sys/crocksdb/c.cc @@ -48,7 +48,7 @@ #include "rocksdb/table_properties.h" #include "rocksdb/types.h" #include "rocksdb/universal_compaction.h" -#include "rocksdb/utilities/backupable_db.h" +#include "rocksdb/utilities/backup_engine.h" #include "rocksdb/utilities/checkpoint.h" #include "rocksdb/utilities/db_ttl.h" #include "rocksdb/utilities/debug.h" @@ -75,8 +75,8 @@ #endif using rocksdb::BackgroundErrorReason; -using rocksdb::BackupableDBOptions; using rocksdb::BackupEngine; +using rocksdb::BackupEngineOptions; using rocksdb::BackupInfo; using rocksdb::BlockBasedTableOptions; using rocksdb::BlockCipher; @@ -524,41 +524,10 @@ struct crocksdb_filterpolicy_t : public FilterPolicy { void* state_; void (*destructor_)(void*); const char* (*name_)(void*); - char* (*create_)(void*, const char* const* key_array, - const size_t* key_length_array, int num_keys, - size_t* filter_length); - unsigned char (*key_match_)(void*, const char* key, size_t length, - const char* filter, size_t filter_length); - void (*delete_filter_)(void*, const char* filter, size_t filter_length); virtual ~crocksdb_filterpolicy_t() { (*destructor_)(state_); } virtual const char* Name() const override { return (*name_)(state_); } - - virtual void CreateFilter(const Slice* keys, int n, - std::string* dst) const override { - std::vector key_pointers(n); - std::vector key_sizes(n); - for (int i = 0; i < n; i++) { - key_pointers[i] = keys[i].data(); - key_sizes[i] = keys[i].size(); - } - size_t len; - char* filter = (*create_)(state_, &key_pointers[0], &key_sizes[0], n, &len); - dst->append(filter, len); - - if (delete_filter_ != nullptr) { - (*delete_filter_)(state_, filter, len); - } else { - free(filter); - } - } - - virtual bool KeyMayMatch(const Slice& key, - const Slice& filter) const override { - return (*key_match_)(state_, key.data(), key.size(), filter.data(), - filter.size()); - } }; struct crocksdb_mergeoperator_t : public MergeOperator { @@ -853,7 +822,7 @@ crocksdb_backup_engine_t* crocksdb_backup_engine_open( const crocksdb_options_t* options, const char* path, char** errptr) { BackupEngine* be; if (SaveError(errptr, BackupEngine::Open(options->rep.env, - BackupableDBOptions(path), &be))) { + BackupEngineOptions(path), &be))) { return nullptr; } crocksdb_backup_engine_t* result = new crocksdb_backup_engine_t; @@ -1189,7 +1158,8 @@ void crocksdb_write_multi_batch(crocksdb_t* db, for (size_t i = 0; i < batch_size; i++) { ws.push_back(&batches[i]->rep); } - SaveError(errptr, db->rep->MultiBatchWrite(options->rep, std::move(ws))); + SaveError(errptr, + db->rep->MultiBatchWrite(options->rep, std::move(ws), nullptr)); } void crocksdb_write_multi_batch_callback( @@ -2097,14 +2067,6 @@ void crocksdb_block_based_options_set_block_cache( } } -void crocksdb_block_based_options_set_block_cache_compressed( - crocksdb_block_based_table_options_t* options, - crocksdb_cache_t* block_cache_compressed) { - if (block_cache_compressed) { - options->rep.block_cache_compressed = block_cache_compressed->rep; - } -} - void crocksdb_block_based_options_set_whole_key_filtering( crocksdb_block_based_table_options_t* options, unsigned char v) { options->rep.whole_key_filtering = v; @@ -2120,11 +2082,6 @@ void crocksdb_block_based_options_set_index_type( options->rep.index_type = static_cast(v); } -void crocksdb_block_based_options_set_hash_index_allow_collision( - crocksdb_block_based_table_options_t* options, unsigned char v) { - options->rep.hash_index_allow_collision = v; -} - void crocksdb_block_based_options_set_optimize_filters_for_memory( crocksdb_block_based_table_options_t* options, unsigned char v) { options->rep.optimize_filters_for_memory = v; @@ -3068,11 +3025,6 @@ void crocksdb_options_set_is_fd_close_on_exec(crocksdb_options_t* opt, opt->rep.is_fd_close_on_exec = v; } -void crocksdb_options_set_skip_log_error_on_recovery(crocksdb_options_t* opt, - unsigned char v) { - opt->rep.skip_log_error_on_recovery = v; -} - void crocksdb_options_set_stats_dump_period_sec(crocksdb_options_t* opt, unsigned int v) { opt->rep.stats_dump_period_sec = v; @@ -3197,16 +3149,6 @@ int crocksdb_options_get_max_background_compactions( return opt->rep.max_background_compactions; } -void crocksdb_options_set_base_background_compactions(crocksdb_options_t* opt, - int n) { - opt->rep.base_background_compactions = n; -} - -int crocksdb_options_get_base_background_compactions( - const crocksdb_options_t* opt) { - return opt->rep.base_background_compactions; -} - void crocksdb_options_set_max_background_flushes(crocksdb_options_t* opt, int n) { opt->rep.max_background_flushes = n; @@ -3234,14 +3176,6 @@ void crocksdb_options_set_recycle_log_file_num(crocksdb_options_t* opt, opt->rep.recycle_log_file_num = v; } -void crocksdb_options_set_soft_rate_limit(crocksdb_options_t* opt, double v) { - opt->rep.soft_rate_limit = v; -} - -void crocksdb_options_set_hard_rate_limit(crocksdb_options_t* opt, double v) { - opt->rep.hard_rate_limit = v; -} - void crocksdb_options_set_soft_pending_compaction_bytes_limit( crocksdb_options_t* opt, size_t v) { opt->rep.soft_pending_compaction_bytes_limit = v; @@ -3262,11 +3196,6 @@ size_t crocksdb_options_get_hard_pending_compaction_bytes_limit( return opt->rep.hard_pending_compaction_bytes_limit; } -void crocksdb_options_set_rate_limit_delay_max_milliseconds( - crocksdb_options_t* opt, unsigned int v) { - opt->rep.rate_limit_delay_max_milliseconds = v; -} - void crocksdb_options_set_max_manifest_file_size(crocksdb_options_t* opt, size_t v) { opt->rep.max_manifest_file_size = v; @@ -3609,12 +3538,14 @@ void crocksdb_options_set_track_and_verify_wals_in_manifest( } unsigned char crocksdb_load_latest_options( - const char* dbpath, crocksdb_env_t* env, crocksdb_options_t* db_options, + const char* dbpath, crocksdb_options_t* db_options, crocksdb_column_family_descriptor*** cf_descs, size_t* cf_descs_len, unsigned char ignore_unknown_options, char** errptr) { std::vector tmp_cf_descs; - Status s = rocksdb::LoadLatestOptions(dbpath, env->rep, &db_options->rep, - &tmp_cf_descs, ignore_unknown_options); + rocksdb::ConfigOptions config_options; + config_options.ignore_unknown_options = ignore_unknown_options; + Status s = rocksdb::LoadLatestOptions(rocksdb::ConfigOptions(), dbpath, + &db_options->rep, &tmp_cf_descs); *errptr = nullptr; if (s.IsNotFound()) return false; @@ -3810,17 +3741,12 @@ unsigned char crocksdb_compactionfiltercontext_is_bottommost_level( return context->rep.is_bottommost_level; } -void crocksdb_compactionfiltercontext_file_numbers( - crocksdb_compactionfiltercontext_t* context, const uint64_t** buffer, - size_t* len) { - *buffer = context->rep.file_numbers.data(); - *len = context->rep.file_numbers.size(); -} - -crocksdb_table_properties_t* crocksdb_compactionfiltercontext_table_properties( - crocksdb_compactionfiltercontext_t* context, size_t offset) { - return (crocksdb_table_properties_t*)context->rep.table_properties[offset] - .get(); +crocksdb_table_properties_collection_t* +crocksdb_compactionfiltercontext_input_table_properties( + crocksdb_compactionfiltercontext_t* context) { + return ( + crocksdb_table_properties_collection_t*)(&context->rep + .input_table_properties); } const char* crocksdb_compactionfiltercontext_start_key( @@ -3879,25 +3805,6 @@ crocksdb_comparator_t* crocksdb_comparator_create( void crocksdb_comparator_destroy(crocksdb_comparator_t* cmp) { delete cmp; } -crocksdb_filterpolicy_t* crocksdb_filterpolicy_create( - void* state, void (*destructor)(void*), - char* (*create_filter)(void*, const char* const* key_array, - const size_t* key_length_array, int num_keys, - size_t* filter_length), - unsigned char (*key_may_match)(void*, const char* key, size_t length, - const char* filter, size_t filter_length), - void (*delete_filter)(void*, const char* filter, size_t filter_length), - const char* (*name)(void*)) { - crocksdb_filterpolicy_t* result = new crocksdb_filterpolicy_t; - result->state_ = state; - result->destructor_ = destructor; - result->create_ = create_filter; - result->key_match_ = key_may_match; - result->delete_filter_ = delete_filter; - result->name_ = name; - return result; -} - void crocksdb_filterpolicy_destroy(crocksdb_filterpolicy_t* filter) { delete filter; } @@ -3910,11 +3817,8 @@ struct FilterPolicyWrapper : public crocksdb_filterpolicy_t { std::string full_name_; ~FilterPolicyWrapper() override { delete rep_; } const char* Name() const override { return full_name_.c_str(); } - void CreateFilter(const Slice* keys, int n, std::string* dst) const override { - return rep_->CreateFilter(keys, n, dst); - } - bool KeyMayMatch(const Slice& key, const Slice& filter) const override { - return rep_->KeyMayMatch(key, filter); + const char* CompatibilityName() const override { + return rep_->CompatibilityName(); } // No need to override GetFilterBitsBuilder if this one is overridden FilterBitsBuilder* GetBuilderWithContext( @@ -3938,7 +3842,6 @@ crocksdb_filterpolicy_t* crocksdb_filterpolicy_create_bloom_format( wrapper->full_name_ += ".FullBloom"; } wrapper->state_ = nullptr; - wrapper->delete_filter_ = nullptr; wrapper->destructor_ = &FilterPolicyWrapper::DoNothing; return wrapper; } @@ -3961,7 +3864,6 @@ crocksdb_filterpolicy_t* crocksdb_filterpolicy_create_ribbon( wrapper->full_name_ = wrapper->rep_->Name(); wrapper->full_name_ += ".Ribbon"; wrapper->state_ = nullptr; - wrapper->delete_filter_ = nullptr; wrapper->destructor_ = &FilterPolicyWrapper::DoNothing; return wrapper; } @@ -5821,7 +5723,7 @@ struct ExternalSstFileModifier { auto ioptions = *cfd->ioptions(); auto table_opt = TableReaderOptions(ioptions, desc.options.prefix_extractor, - env_options_, cfd->internal_comparator()); + env_options_, cfd->internal_comparator(), 0); // Get around global seqno check. table_opt.largest_seqno = kMaxSequenceNumber; status = ioptions.table_factory->NewTableReader( diff --git a/librocksdb_sys/crocksdb/crocksdb/c.h b/librocksdb_sys/crocksdb/crocksdb/c.h index 7e8bce09f..622585b95 100644 --- a/librocksdb_sys/crocksdb/crocksdb/c.h +++ b/librocksdb_sys/crocksdb/crocksdb/c.h @@ -780,10 +780,6 @@ extern C_ROCKSDB_LIBRARY_API void crocksdb_block_based_options_set_block_cache( crocksdb_block_based_table_options_t* options, crocksdb_cache_t* block_cache); extern C_ROCKSDB_LIBRARY_API void -crocksdb_block_based_options_set_block_cache_compressed( - crocksdb_block_based_table_options_t* options, - crocksdb_cache_t* block_cache_compressed); -extern C_ROCKSDB_LIBRARY_API void crocksdb_block_based_options_set_whole_key_filtering( crocksdb_block_based_table_options_t*, unsigned char); extern C_ROCKSDB_LIBRARY_API void @@ -1167,7 +1163,7 @@ crocksdb_options_get_sst_partitioner_factory(crocksdb_options_t*); extern C_ROCKSDB_LIBRARY_API void crocksdb_options_set_sst_partitioner_factory( crocksdb_options_t*, crocksdb_sst_partitioner_factory_t*); extern C_ROCKSDB_LIBRARY_API unsigned char crocksdb_load_latest_options( - const char* dbpath, crocksdb_env_t* env, crocksdb_options_t* db_options, + const char* dbpath, crocksdb_options_t* db_options, crocksdb_column_family_descriptor*** cf_descs, size_t* cf_descs_len, unsigned char ignore_unknown_options, char** errptr); @@ -1222,10 +1218,6 @@ extern C_ROCKSDB_LIBRARY_API void crocksdb_options_set_max_background_compactions(crocksdb_options_t*, int); extern C_ROCKSDB_LIBRARY_API int crocksdb_options_get_max_background_compactions(const crocksdb_options_t*); -extern C_ROCKSDB_LIBRARY_API void -crocksdb_options_set_base_background_compactions(crocksdb_options_t*, int); -extern C_ROCKSDB_LIBRARY_API int -crocksdb_options_get_base_background_compactions(const crocksdb_options_t*); extern C_ROCKSDB_LIBRARY_API void crocksdb_options_set_max_background_flushes( crocksdb_options_t*, int); extern C_ROCKSDB_LIBRARY_API int crocksdb_options_get_max_background_flushes( @@ -1238,10 +1230,6 @@ extern C_ROCKSDB_LIBRARY_API void crocksdb_options_set_keep_log_file_num( crocksdb_options_t*, size_t); extern C_ROCKSDB_LIBRARY_API void crocksdb_options_set_recycle_log_file_num( crocksdb_options_t*, size_t); -extern C_ROCKSDB_LIBRARY_API void crocksdb_options_set_soft_rate_limit( - crocksdb_options_t*, double); -extern C_ROCKSDB_LIBRARY_API void crocksdb_options_set_hard_rate_limit( - crocksdb_options_t*, double); extern C_ROCKSDB_LIBRARY_API void crocksdb_options_set_soft_pending_compaction_bytes_limit( crocksdb_options_t* opt, size_t v); @@ -1254,9 +1242,6 @@ crocksdb_options_set_hard_pending_compaction_bytes_limit( extern C_ROCKSDB_LIBRARY_API size_t crocksdb_options_get_hard_pending_compaction_bytes_limit( crocksdb_options_t* opt); -extern C_ROCKSDB_LIBRARY_API void -crocksdb_options_set_rate_limit_delay_max_milliseconds(crocksdb_options_t*, - unsigned int); extern C_ROCKSDB_LIBRARY_API void crocksdb_options_set_max_manifest_file_size( crocksdb_options_t*, size_t); extern C_ROCKSDB_LIBRARY_API void crocksdb_options_set_table_cache_numshardbits( @@ -1293,9 +1278,6 @@ extern C_ROCKSDB_LIBRARY_API void crocksdb_options_set_allow_mmap_writes( crocksdb_options_t*, unsigned char); extern C_ROCKSDB_LIBRARY_API void crocksdb_options_set_is_fd_close_on_exec( crocksdb_options_t*, unsigned char); -extern C_ROCKSDB_LIBRARY_API void -crocksdb_options_set_skip_log_error_on_recovery(crocksdb_options_t*, - unsigned char); extern C_ROCKSDB_LIBRARY_API void crocksdb_options_set_stats_dump_period_sec( crocksdb_options_t*, unsigned int); extern C_ROCKSDB_LIBRARY_API void crocksdb_options_set_stats_persist_period_sec( @@ -1549,13 +1531,9 @@ extern C_ROCKSDB_LIBRARY_API unsigned char crocksdb_compactionfiltercontext_is_bottommost_level( crocksdb_compactionfiltercontext_t* context); -extern C_ROCKSDB_LIBRARY_API void crocksdb_compactionfiltercontext_file_numbers( - crocksdb_compactionfiltercontext_t* context, const uint64_t** buffer, - size_t* len); - -extern C_ROCKSDB_LIBRARY_API crocksdb_table_properties_t* -crocksdb_compactionfiltercontext_table_properties( - crocksdb_compactionfiltercontext_t* context, size_t offset); +extern C_ROCKSDB_LIBRARY_API crocksdb_table_properties_collection_t* +crocksdb_compactionfiltercontext_input_table_properties( + crocksdb_compactionfiltercontext_t* context); extern C_ROCKSDB_LIBRARY_API uint32_t crocksdb_compactionfiltercontext_reason( crocksdb_compactionfiltercontext_t* context); @@ -1584,16 +1562,6 @@ extern C_ROCKSDB_LIBRARY_API void crocksdb_comparator_destroy( /* Filter policy */ -extern C_ROCKSDB_LIBRARY_API crocksdb_filterpolicy_t* -crocksdb_filterpolicy_create( - void* state, void (*destructor)(void*), - char* (*create_filter)(void*, const char* const* key_array, - const size_t* key_length_array, int num_keys, - size_t* filter_length), - unsigned char (*key_may_match)(void*, const char* key, size_t length, - const char* filter, size_t filter_length), - void (*delete_filter)(void*, const char* filter, size_t filter_length), - const char* (*name)(void*)); extern C_ROCKSDB_LIBRARY_API void crocksdb_filterpolicy_destroy( crocksdb_filterpolicy_t*); diff --git a/librocksdb_sys/libtitan_sys/titan b/librocksdb_sys/libtitan_sys/titan index ff85fd19e..0e453240c 160000 --- a/librocksdb_sys/libtitan_sys/titan +++ b/librocksdb_sys/libtitan_sys/titan @@ -1 +1 @@ -Subproject commit ff85fd19e6268c57b3b01c45aeae011299c1d542 +Subproject commit 0e453240c28632342bc46ce67a9df5c067764aef diff --git a/librocksdb_sys/rocksdb b/librocksdb_sys/rocksdb index 45509f0f5..620dcb341 160000 --- a/librocksdb_sys/rocksdb +++ b/librocksdb_sys/rocksdb @@ -1 +1 @@ -Subproject commit 45509f0f530ad370863876fc1ee95ccf85bfe96d +Subproject commit 620dcb341c33161cb3ef29e8b146620cfd467564 diff --git a/librocksdb_sys/src/generate.py b/librocksdb_sys/src/generate.py index fa5cfdb85..051fc12eb 100755 --- a/librocksdb_sys/src/generate.py +++ b/librocksdb_sys/src/generate.py @@ -12,13 +12,13 @@ ( '../rocksdb/include/rocksdb/statistics.h', re.compile(r'enum Tickers .* {'), - re.compile(r'\s*\w(_\w)*.*,'), + re.compile(r'\s*\w(_\w)*.*'), re.compile(r'};\s*'), ), ( '../libtitan_sys/titan/include/titan/statistics.h', re.compile(r'enum TickerType .* {'), - re.compile(r'\s*\w(_\w)*.*,'), + re.compile(r'\s*\w(_\w)*.*'), re.compile(r'};\s*'), ), ], @@ -29,13 +29,13 @@ ( '../rocksdb/include/rocksdb/statistics.h', re.compile(r'enum Histograms .* {'), - re.compile(r'\s*\w(_\w)*.*,'), + re.compile(r'\s*\w(_\w)*.*'), re.compile(r'};\s*'), ), ( '../libtitan_sys/titan/include/titan/statistics.h', re.compile(r'enum HistogramType .* {'), - re.compile(r'\s*\w(_\w)*.*,'), + re.compile(r'\s*\w(_\w)*.*'), re.compile(r'};\s*'), ), ], @@ -69,6 +69,8 @@ break if not cpp_entry_pattern.match(line): continue + # remove return line + line = line.strip() tokens = line.split(',')[0].split('=') if len(tokens) == 1: name = tokens[0].strip(' ') diff --git a/librocksdb_sys/src/generated.rs b/librocksdb_sys/src/generated.rs index 346fd0d2f..97f9b2bd0 100644 --- a/librocksdb_sys/src/generated.rs +++ b/librocksdb_sys/src/generated.rs @@ -12,207 +12,243 @@ pub enum DBStatisticsTickerType { BlockCacheIndexHit = 5, BlockCacheIndexAdd = 6, BlockCacheIndexBytesInsert = 7, - BlockCacheIndexBytesEvict = 8, - BlockCacheFilterMiss = 9, - BlockCacheFilterHit = 10, - BlockCacheFilterAdd = 11, - BlockCacheFilterBytesInsert = 12, - BlockCacheFilterBytesEvict = 13, - BlockCacheDataMiss = 14, - BlockCacheDataHit = 15, - BlockCacheDataAdd = 16, - BlockCacheDataBytesInsert = 17, - BlockCacheBytesRead = 18, - BlockCacheBytesWrite = 19, - BloomFilterUseful = 20, - BloomFilterFullPositive = 21, - BloomFilterFullTruePositive = 22, - BloomFilterMicros = 23, - PersistentCacheHit = 24, - PersistentCacheMiss = 25, - SimBlockCacheHit = 26, - SimBlockCacheMiss = 27, - MemtableHit = 28, - MemtableMiss = 29, - GetHitL0 = 30, - GetHitL1 = 31, - GetHitL2AndUp = 32, - CompactionKeyDropNewerEntry = 33, - CompactionKeyDropObsolete = 34, - CompactionKeyDropRangeDel = 35, - CompactionKeyDropUser = 36, - CompactionRangeDelDropObsolete = 37, - CompactionOptimizedDelDropObsolete = 38, - CompactionCancelled = 39, - NumberKeysWritten = 40, - NumberKeysRead = 41, - NumberKeysUpdated = 42, - BytesWritten = 43, - BytesRead = 44, - NumberDbSeek = 45, - NumberDbNext = 46, - NumberDbPrev = 47, - NumberDbSeekFound = 48, - NumberDbNextFound = 49, - NumberDbPrevFound = 50, - IterBytesRead = 51, - NoFileCloses = 52, - NoFileOpens = 53, - NoFileErrors = 54, - StallL0SlowdownMicros = 55, - StallMemtableCompactionMicros = 56, - StallL0NumFilesMicros = 57, - StallMicros = 58, - DbMutexWaitMicros = 59, - RateLimitDelayMillis = 60, - NoIterators = 61, - NumberMultigetCalls = 62, - NumberMultigetKeysRead = 63, - NumberMultigetBytesRead = 64, - NumberFilteredDeletes = 65, - NumberMergeFailures = 66, - BloomFilterPrefixChecked = 67, - BloomFilterPrefixUseful = 68, - NumberOfReseeksInIteration = 69, - GetUpdatesSinceCalls = 70, - BlockCacheCompressedMiss = 71, - BlockCacheCompressedHit = 72, - BlockCacheCompressedAdd = 73, - BlockCacheCompressedAddFailures = 74, - WalFileSynced = 75, - WalFileBytes = 76, - WriteDoneBySelf = 77, - WriteDoneByOther = 78, - WriteTimedout = 79, - WriteWithWal = 80, - CompactReadBytes = 81, - CompactWriteBytes = 82, - FlushWriteBytes = 83, - CompactReadBytesMarked = 84, - CompactReadBytesPeriodic = 85, - CompactReadBytesTtl = 86, - CompactWriteBytesMarked = 87, - CompactWriteBytesPeriodic = 88, - CompactWriteBytesTtl = 89, - NumberDirectLoadTableProperties = 90, - NumberSuperversionAcquires = 91, - NumberSuperversionReleases = 92, - NumberSuperversionCleanups = 93, - NumberBlockCompressed = 94, - NumberBlockDecompressed = 95, - NumberBlockNotCompressed = 96, - MergeOperationTotalTime = 97, - FilterOperationTotalTime = 98, - RowCacheHit = 99, - RowCacheMiss = 100, - ReadAmpEstimateUsefulBytes = 101, - ReadAmpTotalReadBytes = 102, - NumberRateLimiterDrains = 103, - NumberIterSkip = 104, - BlobDbNumPut = 105, - BlobDbNumWrite = 106, - BlobDbNumGet = 107, - BlobDbNumMultiget = 108, - BlobDbNumSeek = 109, - BlobDbNumNext = 110, - BlobDbNumPrev = 111, - BlobDbNumKeysWritten = 112, - BlobDbNumKeysRead = 113, - BlobDbBytesWritten = 114, - BlobDbBytesRead = 115, - BlobDbWriteInlined = 116, - BlobDbWriteInlinedTtl = 117, - BlobDbWriteBlob = 118, - BlobDbWriteBlobTtl = 119, - BlobDbBlobFileBytesWritten = 120, - BlobDbBlobFileBytesRead = 121, - BlobDbBlobFileSynced = 122, - BlobDbBlobIndexExpiredCount = 123, - BlobDbBlobIndexExpiredSize = 124, - BlobDbBlobIndexEvictedCount = 125, - BlobDbBlobIndexEvictedSize = 126, - BlobDbGcNumFiles = 127, - BlobDbGcNumNewFiles = 128, - BlobDbGcFailures = 129, - BlobDbGcNumKeysOverwritten = 130, - BlobDbGcNumKeysExpired = 131, - BlobDbGcNumKeysRelocated = 132, - BlobDbGcBytesOverwritten = 133, - BlobDbGcBytesExpired = 134, - BlobDbGcBytesRelocated = 135, - BlobDbFifoNumFilesEvicted = 136, - BlobDbFifoNumKeysEvicted = 137, - BlobDbFifoBytesEvicted = 138, - TxnPrepareMutexOverhead = 139, - TxnOldCommitMapMutexOverhead = 140, - TxnDuplicateKeyOverhead = 141, - TxnSnapshotMutexOverhead = 142, - TxnGetTryAgain = 143, - NumberMultigetKeysFound = 144, - NoIteratorCreated = 145, - NoIteratorDeleted = 146, - BlockCacheCompressionDictMiss = 147, - BlockCacheCompressionDictHit = 148, - BlockCacheCompressionDictAdd = 149, - BlockCacheCompressionDictBytesInsert = 150, - BlockCacheCompressionDictBytesEvict = 151, - BlockCacheAddRedundant = 152, - BlockCacheIndexAddRedundant = 153, - BlockCacheFilterAddRedundant = 154, - BlockCacheDataAddRedundant = 155, - BlockCacheCompressionDictAddRedundant = 156, - FilesMarkedTrash = 157, - FilesDeletedImmediately = 158, - ErrorHandlerBgErrorCount = 159, - ErrorHandlerBgIoErrorCount = 160, - ErrorHandlerBgRetryableIoErrorCount = 161, - ErrorHandlerAutoresumeCount = 162, - ErrorHandlerAutoresumeRetryTotalCount = 163, - ErrorHandlerAutoresumeSuccessCount = 164, - MemtablePayloadBytesAtFlush = 165, - MemtableGarbageBytesAtFlush = 166, - SecondaryCacheHits = 167, - VerifyChecksumReadBytes = 168, - BackupReadBytes = 169, - BackupWriteBytes = 170, - RemoteCompactReadBytes = 171, - RemoteCompactWriteBytes = 172, - HotFileReadBytes = 173, - WarmFileReadBytes = 174, - ColdFileReadBytes = 175, - HotFileReadCount = 176, - WarmFileReadCount = 177, - ColdFileReadCount = 178, - TitanNumGet = 179, - TitanNumSeek = 180, - TitanNumNext = 181, - TitanNumPrev = 182, - TitanBlobFileNumKeysWritten = 183, - TitanBlobFileNumKeysRead = 184, - TitanBlobFileBytesWritten = 185, - TitanBlobFileBytesRead = 186, - TitanBlobFileSynced = 187, - TitanGcNumFiles = 188, - TitanGcNumNewFiles = 189, - TitanGcNumKeysOverwritten = 190, - TitanGcNumKeysRelocated = 191, - TitanGcNumKeysFallback = 192, - TitanGcBytesOverwritten = 193, - TitanGcBytesRelocated = 194, - TitanGcBytesFallback = 195, - TitanGcBytesWritten = 196, - TitanGcBytesRead = 197, - TitanBlobCacheHit = 198, - TitanBlobCacheMiss = 199, - TitanGcDiscardable = 200, - TitanGcSmallFile = 201, - TitanGcLevelMergeMark = 202, - TitanGcLevelMergeDelete = 203, - TitanGcNoNeed = 204, - TitanGcRemain = 205, - TitanGcFailure = 206, - TitanGcSuccess = 207, - TitanGcTriggerNext = 208, + BlockCacheFilterMiss = 8, + BlockCacheFilterHit = 9, + BlockCacheFilterAdd = 10, + BlockCacheFilterBytesInsert = 11, + BlockCacheDataMiss = 12, + BlockCacheDataHit = 13, + BlockCacheDataAdd = 14, + BlockCacheDataBytesInsert = 15, + BlockCacheBytesRead = 16, + BlockCacheBytesWrite = 17, + BloomFilterUseful = 18, + BloomFilterFullPositive = 19, + BloomFilterFullTruePositive = 20, + PersistentCacheHit = 21, + PersistentCacheMiss = 22, + SimBlockCacheHit = 23, + SimBlockCacheMiss = 24, + MemtableHit = 25, + MemtableMiss = 26, + GetHitL0 = 27, + GetHitL1 = 28, + GetHitL2AndUp = 29, + CompactionKeyDropNewerEntry = 30, + CompactionKeyDropObsolete = 31, + CompactionKeyDropRangeDel = 32, + CompactionKeyDropUser = 33, + CompactionRangeDelDropObsolete = 34, + CompactionOptimizedDelDropObsolete = 35, + CompactionCancelled = 36, + NumberKeysWritten = 37, + NumberKeysRead = 38, + NumberKeysUpdated = 39, + BytesWritten = 40, + BytesRead = 41, + NumberDbSeek = 42, + NumberDbNext = 43, + NumberDbPrev = 44, + NumberDbSeekFound = 45, + NumberDbNextFound = 46, + NumberDbPrevFound = 47, + IterBytesRead = 48, + NoFileOpens = 49, + NoFileErrors = 50, + StallMicros = 51, + DbMutexWaitMicros = 52, + NumberMultigetCalls = 53, + NumberMultigetKeysRead = 54, + NumberMultigetBytesRead = 55, + NumberMergeFailures = 56, + BloomFilterPrefixChecked = 57, + BloomFilterPrefixUseful = 58, + BloomFilterPrefixTruePositive = 59, + NumberOfReseeksInIteration = 60, + GetUpdatesSinceCalls = 61, + WalFileSynced = 62, + WalFileBytes = 63, + WriteDoneBySelf = 64, + WriteDoneByOther = 65, + WriteWithWal = 66, + CompactReadBytes = 67, + CompactWriteBytes = 68, + FlushWriteBytes = 69, + CompactReadBytesMarked = 70, + CompactReadBytesPeriodic = 71, + CompactReadBytesTtl = 72, + CompactWriteBytesMarked = 73, + CompactWriteBytesPeriodic = 74, + CompactWriteBytesTtl = 75, + NumberDirectLoadTableProperties = 76, + NumberSuperversionAcquires = 77, + NumberSuperversionReleases = 78, + NumberSuperversionCleanups = 79, + NumberBlockCompressed = 80, + NumberBlockDecompressed = 81, + NumberBlockNotCompressed = 82, + MergeOperationTotalTime = 83, + FilterOperationTotalTime = 84, + CompactionCpuTotalTime = 85, + RowCacheHit = 86, + RowCacheMiss = 87, + ReadAmpEstimateUsefulBytes = 88, + ReadAmpTotalReadBytes = 89, + NumberRateLimiterDrains = 90, + NumberIterSkip = 91, + BlobDbNumPut = 92, + BlobDbNumWrite = 93, + BlobDbNumGet = 94, + BlobDbNumMultiget = 95, + BlobDbNumSeek = 96, + BlobDbNumNext = 97, + BlobDbNumPrev = 98, + BlobDbNumKeysWritten = 99, + BlobDbNumKeysRead = 100, + BlobDbBytesWritten = 101, + BlobDbBytesRead = 102, + BlobDbWriteInlined = 103, + BlobDbWriteInlinedTtl = 104, + BlobDbWriteBlob = 105, + BlobDbWriteBlobTtl = 106, + BlobDbBlobFileBytesWritten = 107, + BlobDbBlobFileBytesRead = 108, + BlobDbBlobFileSynced = 109, + BlobDbBlobIndexExpiredCount = 110, + BlobDbBlobIndexExpiredSize = 111, + BlobDbBlobIndexEvictedCount = 112, + BlobDbBlobIndexEvictedSize = 113, + BlobDbGcNumFiles = 114, + BlobDbGcNumNewFiles = 115, + BlobDbGcFailures = 116, + BlobDbGcNumKeysRelocated = 117, + BlobDbGcBytesRelocated = 118, + BlobDbFifoNumFilesEvicted = 119, + BlobDbFifoNumKeysEvicted = 120, + BlobDbFifoBytesEvicted = 121, + TxnPrepareMutexOverhead = 122, + TxnOldCommitMapMutexOverhead = 123, + TxnDuplicateKeyOverhead = 124, + TxnSnapshotMutexOverhead = 125, + TxnGetTryAgain = 126, + NumberMultigetKeysFound = 127, + NoIteratorCreated = 128, + NoIteratorDeleted = 129, + BlockCacheCompressionDictMiss = 130, + BlockCacheCompressionDictHit = 131, + BlockCacheCompressionDictAdd = 132, + BlockCacheCompressionDictBytesInsert = 133, + BlockCacheAddRedundant = 134, + BlockCacheIndexAddRedundant = 135, + BlockCacheFilterAddRedundant = 136, + BlockCacheDataAddRedundant = 137, + BlockCacheCompressionDictAddRedundant = 138, + FilesMarkedTrash = 139, + FilesDeletedFromTrashQueue = 140, + FilesDeletedImmediately = 141, + ErrorHandlerBgErrorCount = 142, + ErrorHandlerBgErrorCountMisspelled = 143, + ErrorHandlerBgIoErrorCount = 144, + ErrorHandlerBgIoErrorCountMisspelled = 145, + ErrorHandlerBgRetryableIoErrorCount = 146, + ErrorHandlerBgRetryableIoErrorCountMisspelled = 147, + ErrorHandlerAutoresumeCount = 148, + ErrorHandlerAutoresumeRetryTotalCount = 149, + ErrorHandlerAutoresumeSuccessCount = 150, + MemtablePayloadBytesAtFlush = 151, + MemtableGarbageBytesAtFlush = 152, + SecondaryCacheHits = 153, + VerifyChecksumReadBytes = 154, + BackupReadBytes = 155, + BackupWriteBytes = 156, + RemoteCompactReadBytes = 157, + RemoteCompactWriteBytes = 158, + HotFileReadBytes = 159, + WarmFileReadBytes = 160, + ColdFileReadBytes = 161, + HotFileReadCount = 162, + WarmFileReadCount = 163, + ColdFileReadCount = 164, + LastLevelReadBytes = 165, + LastLevelReadCount = 166, + NonLastLevelReadBytes = 167, + NonLastLevelReadCount = 168, + LastLevelSeekFiltered = 169, + LastLevelSeekFilterMatch = 170, + LastLevelSeekData = 171, + LastLevelSeekDataUsefulNoFilter = 172, + LastLevelSeekDataUsefulFilterMatch = 173, + NonLastLevelSeekFiltered = 174, + NonLastLevelSeekFilterMatch = 175, + NonLastLevelSeekData = 176, + NonLastLevelSeekDataUsefulNoFilter = 177, + NonLastLevelSeekDataUsefulFilterMatch = 178, + BlockChecksumComputeCount = 179, + BlockChecksumMismatchCount = 180, + MultigetCoroutineCount = 181, + BlobDbCacheMiss = 182, + BlobDbCacheHit = 183, + BlobDbCacheAdd = 184, + BlobDbCacheAddFailures = 185, + BlobDbCacheBytesRead = 186, + BlobDbCacheBytesWrite = 187, + ReadAsyncMicros = 188, + AsyncReadErrorCount = 189, + SecondaryCacheFilterHits = 190, + SecondaryCacheIndexHits = 191, + SecondaryCacheDataHits = 192, + TableOpenPrefetchTailMiss = 193, + TableOpenPrefetchTailHit = 194, + TimestampFilterTableChecked = 195, + TimestampFilterTableFiltered = 196, + BytesCompressedFrom = 197, + BytesCompressedTo = 198, + BytesCompressionBypassed = 199, + BytesCompressionRejected = 200, + NumberBlockCompressionBypassed = 201, + NumberBlockCompressionRejected = 202, + BytesDecompressedFrom = 203, + BytesDecompressedTo = 204, + ReadaheadTrimmed = 205, + FifoMaxSizeCompactions = 206, + FifoTtlCompactions = 207, + PrefetchBytes = 208, + PrefetchBytesUseful = 209, + PrefetchHits = 210, + CompressedSecondaryCacheDummyHits = 211, + CompressedSecondaryCacheHits = 212, + CompressedSecondaryCachePromotions = 213, + CompressedSecondaryCachePromotionSkips = 214, + TitanNumGet = 215, + TitanNumSeek = 216, + TitanNumNext = 217, + TitanNumPrev = 218, + TitanBlobFileNumKeysWritten = 219, + TitanBlobFileNumKeysRead = 220, + TitanBlobFileBytesWritten = 221, + TitanBlobFileBytesRead = 222, + TitanBlobFileSynced = 223, + TitanGcNumFiles = 224, + TitanGcNumNewFiles = 225, + TitanGcNumKeysOverwritten = 226, + TitanGcNumKeysRelocated = 227, + TitanGcNumKeysFallback = 228, + TitanGcBytesOverwritten = 229, + TitanGcBytesRelocated = 230, + TitanGcBytesFallback = 231, + TitanGcBytesWritten = 232, + TitanGcBytesRead = 233, + TitanBlobCacheHit = 234, + TitanBlobCacheMiss = 235, + TitanGcDiscardable = 236, + TitanGcSmallFile = 237, + TitanGcLevelMergeMark = 238, + TitanGcLevelMergeDelete = 239, + TitanGcNoNeed = 240, + TitanGcRemain = 241, + TitanGcFailure = 242, + TitanGcSuccess = 243, + TitanGcTriggerNext = 244, } #[derive(Copy, Clone, Debug, Eq, PartialEq)] #[repr(u32)] @@ -231,57 +267,65 @@ pub enum DBStatisticsHistogramType { ReadBlockCompactionMicros = 11, ReadBlockGetMicros = 12, WriteRawBlockMicros = 13, - StallL0SlowdownCount = 14, - StallMemtableCompactionCount = 15, - StallL0NumFilesCount = 16, - HardRateLimitDelayCount = 17, - SoftRateLimitDelayCount = 18, - NumFilesInSingleCompaction = 19, - DbSeek = 20, - WriteStall = 21, - SstReadMicros = 22, - NumSubcompactionsScheduled = 23, - BytesPerRead = 24, - BytesPerWrite = 25, - BytesPerMultiget = 26, - BytesCompressed = 27, - BytesDecompressed = 28, - CompressionTimesNanos = 29, - DecompressionTimesNanos = 30, - ReadNumMergeOperands = 31, - BlobDbKeySize = 32, - BlobDbValueSize = 33, - BlobDbWriteMicros = 34, - BlobDbGetMicros = 35, - BlobDbMultigetMicros = 36, - BlobDbSeekMicros = 37, - BlobDbNextMicros = 38, - BlobDbPrevMicros = 39, - BlobDbBlobFileWriteMicros = 40, - BlobDbBlobFileReadMicros = 41, - BlobDbBlobFileSyncMicros = 42, - BlobDbGcMicros = 43, - BlobDbCompressionMicros = 44, - BlobDbDecompressionMicros = 45, - FlushTime = 46, - SstBatchSize = 47, - DbWriteWalTime = 48, - NumIndexAndFilterBlocksReadPerLevel = 49, - NumDataBlocksReadPerLevel = 50, - NumSstReadPerLevel = 51, - ErrorHandlerAutoresumeRetryCount = 52, - TitanKeySize = 53, - TitanValueSize = 54, - TitanGetMicros = 55, - TitanSeekMicros = 56, - TitanNextMicros = 57, - TitanPrevMicros = 58, - TitanBlobFileWriteMicros = 59, - TitanBlobFileReadMicros = 60, - TitanBlobFileSyncMicros = 61, - TitanManifestFileSyncMicros = 62, - TitanGcMicros = 63, - TitanGcInputFileSize = 64, - TitanGcOutputFileSize = 65, - TitanIterTouchBlobFileCount = 66, + NumFilesInSingleCompaction = 14, + DbSeek = 15, + WriteStall = 16, + SstReadMicros = 17, + FileReadFlushMicros = 18, + FileReadCompactionMicros = 19, + FileReadDbOpenMicros = 20, + FileReadGetMicros = 21, + FileReadMultigetMicros = 22, + FileReadDbIteratorMicros = 23, + FileReadVerifyDbChecksumMicros = 24, + FileReadVerifyFileChecksumsMicros = 25, + NumSubcompactionsScheduled = 26, + BytesPerRead = 27, + BytesPerWrite = 28, + BytesPerMultiget = 29, + BytesCompressed = 30, + BytesDecompressed = 31, + CompressionTimesNanos = 32, + DecompressionTimesNanos = 33, + ReadNumMergeOperands = 34, + BlobDbKeySize = 35, + BlobDbValueSize = 36, + BlobDbWriteMicros = 37, + BlobDbGetMicros = 38, + BlobDbMultigetMicros = 39, + BlobDbSeekMicros = 40, + BlobDbNextMicros = 41, + BlobDbPrevMicros = 42, + BlobDbBlobFileWriteMicros = 43, + BlobDbBlobFileReadMicros = 44, + BlobDbBlobFileSyncMicros = 45, + BlobDbCompressionMicros = 46, + BlobDbDecompressionMicros = 47, + FlushTime = 48, + SstBatchSize = 49, + DbWriteWalTime = 50, + NumIndexAndFilterBlocksReadPerLevel = 51, + NumSstReadPerLevel = 52, + ErrorHandlerAutoresumeRetryCount = 53, + AsyncReadBytes = 54, + PollWaitMicros = 55, + PrefetchedBytesDiscarded = 56, + MultigetIoBatchSize = 57, + NumLevelReadPerMultiget = 58, + AsyncPrefetchAbortMicros = 59, + TableOpenPrefetchTailReadBytes = 60, + TitanKeySize = 61, + TitanValueSize = 62, + TitanGetMicros = 63, + TitanSeekMicros = 64, + TitanNextMicros = 65, + TitanPrevMicros = 66, + TitanBlobFileWriteMicros = 67, + TitanBlobFileReadMicros = 68, + TitanBlobFileSyncMicros = 69, + TitanManifestFileSyncMicros = 70, + TitanGcMicros = 71, + TitanGcInputFileSize = 72, + TitanGcOutputFileSize = 73, + TitanIterTouchBlobFileCount = 74, } diff --git a/librocksdb_sys/src/lib.rs b/librocksdb_sys/src/lib.rs index 67a034689..b5014b830 100644 --- a/librocksdb_sys/src/lib.rs +++ b/librocksdb_sys/src/lib.rs @@ -197,9 +197,9 @@ pub struct DBFileSystemInspectorInstance(c_void); #[derive(Copy, Clone, Debug, Eq, PartialEq)] #[repr(C)] pub enum WriteStallCondition { - Normal = 0, - Delayed = 1, - Stopped = 2, + Delayed = 0, + Stopped = 1, + Normal = 2, } mod generated; @@ -680,10 +680,6 @@ extern "C" { block_options: *mut DBBlockBasedTableOptions, v: IndexType, ); - pub fn crocksdb_block_based_options_set_hash_index_allow_collision( - block_options: *mut DBBlockBasedTableOptions, - v: c_uchar, - ); pub fn crocksdb_block_based_options_set_optimize_filters_for_memory( block_options: *mut DBBlockBasedTableOptions, v: c_uchar, @@ -716,10 +712,6 @@ extern "C" { block_options: *mut DBBlockBasedTableOptions, block_cache: *mut DBCache, ); - pub fn crocksdb_block_based_options_set_block_cache_compressed( - block_options: *mut DBBlockBasedTableOptions, - block_cache_compressed: *mut DBCache, - ); pub fn crocksdb_block_based_options_set_whole_key_filtering( ck_options: *mut DBBlockBasedTableOptions, doit: bool, @@ -882,11 +874,6 @@ extern "C" { max_bg_compactions: c_int, ); pub fn crocksdb_options_get_max_background_compactions(options: *const Options) -> c_int; - pub fn crocksdb_options_set_base_background_compactions( - options: *mut Options, - base_bg_compactions: c_int, - ); - pub fn crocksdb_options_get_base_background_compactions(options: *const Options) -> c_int; pub fn crocksdb_options_set_max_background_flushes( options: *mut Options, max_bg_flushes: c_int, @@ -995,7 +982,6 @@ extern "C" { pub fn crocksdb_load_latest_options( dbpath: *const c_char, - env: *mut DBEnv, db_options: *const Options, cf_descs: *const *mut *mut ColumnFamilyDescriptor, cf_descs_len: *mut size_t, @@ -1815,15 +1801,9 @@ extern "C" { pub fn crocksdb_compactionfiltercontext_is_bottommost_level( context: *const DBCompactionFilterContext, ) -> bool; - pub fn crocksdb_compactionfiltercontext_file_numbers( - context: *const DBCompactionFilterContext, - buffer: *mut *const u64, - len: *mut usize, - ); - pub fn crocksdb_compactionfiltercontext_table_properties( + pub fn crocksdb_compactionfiltercontext_input_table_properties( context: *const DBCompactionFilterContext, - offset: usize, - ) -> *const DBTableProperties; + ) -> *const DBTablePropertiesCollection; pub fn crocksdb_compactionfiltercontext_start_key( context: *const DBCompactionFilterContext, key_len: *mut size_t, diff --git a/src/compaction_filter.rs b/src/compaction_filter.rs index c2f6fbf77..a60ea80f4 100644 --- a/src/compaction_filter.rs +++ b/src/compaction_filter.rs @@ -1,7 +1,7 @@ use std::ffi::CString; use std::{ptr, slice, usize}; -use crate::table_properties::TableProperties; +use crate::TablePropertiesCollectionView; use crocksdb_ffi::CompactionFilterDecision as RawCompactionFilterDecision; pub use crocksdb_ffi::CompactionFilterValueType; pub use crocksdb_ffi::DBCompactionFilter; @@ -203,24 +203,11 @@ impl CompactionFilterContext { unsafe { crocksdb_ffi::crocksdb_compactionfiltercontext_is_bottommost_level(ctx) } } - pub fn file_numbers(&self) -> &[u64] { + pub fn input_table_properties(&self) -> &TablePropertiesCollectionView { let ctx = &self.0 as *const DBCompactionFilterContext; - let (mut buffer, mut len): (*const u64, usize) = (ptr::null_mut(), 0); unsafe { - crocksdb_ffi::crocksdb_compactionfiltercontext_file_numbers( - ctx, - &mut buffer as *mut *const u64, - &mut len as *mut usize, - ); - slice::from_raw_parts(buffer, len) - } - } - - pub fn table_properties(&self, offset: usize) -> &TableProperties { - let ctx = &self.0 as *const DBCompactionFilterContext; - unsafe { - let raw = crocksdb_ffi::crocksdb_compactionfiltercontext_table_properties(ctx, offset); - TableProperties::from_ptr(raw) + let raw = crocksdb_ffi::crocksdb_compactionfiltercontext_input_table_properties(ctx); + TablePropertiesCollectionView::from_ptr(raw) } } diff --git a/src/perf_context.rs b/src/perf_context.rs index ce9ebd9bc..b3b2d4970 100644 --- a/src/perf_context.rs +++ b/src/perf_context.rs @@ -101,21 +101,36 @@ pub enum PerfFlag { BlockReadCount, BlockReadByte, BlockReadTime, + BlockReadCpuTime, BlockCacheIndexHitCount, + BlockCacheStandaloneHandleCount, + BlockCacheRealHandleCount, IndexBlockReadCount, BlockCacheFilterHitCount, FilterBlockReadCount, CompressionDictBlockReadCount, SecondaryCacheHitCount, + CompressedSecCacheInsertRealCount, + CompressedSecCacheInsertDummyCount, + CompressedSecCacheUncompressedBytes, + CompressedSecCacheCompressedBytes, BlockChecksumTime, BlockDecompressTime, GetReadBytes, MultigetReadBytes, IterReadBytes, + BlobCacheHitCount, + BlobReadCount, + BlobReadByte, + BlobReadTime, + BlobChecksumTime, + BlobDecompressTime, InternalKeySkippedCount, InternalDeleteSkippedCount, InternalRecentSkippedCount, InternalMergeCount, + InternalMergePointLookupCount, + InternalRangeDelReseekCount, GetSnapshotTime, GetFromMemtableTime, GetFromMemtableCount, @@ -176,8 +191,12 @@ pub enum PerfFlag { IterNextCpuNanos, IterPrevCpuNanos, IterSeekCpuNanos, + IterNextCount, + IterPrevCount, + IterSeekCount, EncryptDataNanos, DecryptDataNanos, + NumberAsyncSeek, GetFromTableNanos, UserKeyReturnCount, BlockCacheMissCount, diff --git a/src/rocksdb.rs b/src/rocksdb.rs index e00d87e75..2261899d5 100644 --- a/src/rocksdb.rs +++ b/src/rocksdb.rs @@ -2951,7 +2951,6 @@ pub fn set_external_sst_file_global_seq_no( pub fn load_latest_options( dbpath: &str, - env: &Env, ignore_unknown_options: bool, ) -> Result)>, String> { const ERR_CONVERT_PATH: &str = "Failed to convert path to CString when load latest options"; @@ -2964,7 +2963,6 @@ pub fn load_latest_options( let ok = ffi_try!(crocksdb_load_latest_options( dbpath.as_ptr(), - env.inner, db_options.inner, &raw_cf_descs, &mut cf_descs_len, @@ -3534,27 +3532,23 @@ mod test { let cf_name: &str = "cf_dynamic_level_bytes"; // test when options not exist - assert!(load_latest_options(dbpath, &Env::default(), false) - .unwrap() - .is_none()); + assert!(load_latest_options(dbpath, false).unwrap().is_none()); let mut opts = DBOptions::new(); opts.create_if_missing(true); let mut db = DB::open(opts, dbpath).unwrap(); let mut cf_opts = ColumnFamilyOptions::new(); - cf_opts.set_level_compaction_dynamic_level_bytes(true); + cf_opts.set_level_compaction_dynamic_level_bytes(false); db.create_cf((cf_name.clone(), cf_opts)).unwrap(); let cf_handle = db.cf_handle(cf_name.clone()).unwrap(); let cf_opts = db.get_options_cf(cf_handle); - assert!(cf_opts.get_level_compaction_dynamic_level_bytes()); + assert!(!cf_opts.get_level_compaction_dynamic_level_bytes()); - let (_, cf_descs) = load_latest_options(dbpath, &Env::default(), false) - .unwrap() - .unwrap(); + let (_, cf_descs) = load_latest_options(dbpath, false).unwrap().unwrap(); for cf_desc in cf_descs { - if cf_desc.name() == cf_name { + if cf_desc.name() != cf_name { assert!(cf_desc.options().get_level_compaction_dynamic_level_bytes()); } else { assert!(!cf_desc.options().get_level_compaction_dynamic_level_bytes()); diff --git a/src/rocksdb_options.rs b/src/rocksdb_options.rs index 32a73ae8f..3d40dfd79 100644 --- a/src/rocksdb_options.rs +++ b/src/rocksdb_options.rs @@ -144,14 +144,6 @@ impl BlockBasedOptions { } } - pub fn set_hash_index_allow_collision(&mut self, v: bool) { - unsafe { - crocksdb_ffi::crocksdb_block_based_options_set_hash_index_allow_collision( - self.inner, v as u8, - ); - } - } - pub fn set_partition_filters(&mut self, v: bool) { unsafe { crocksdb_ffi::crocksdb_block_based_options_set_partition_filters(self.inner, v as u8); @@ -1097,16 +1089,6 @@ impl DBOptions { unsafe { crocksdb_ffi::crocksdb_options_get_max_background_compactions(self.inner) } } - pub fn set_base_background_compactions(&mut self, n: c_int) { - unsafe { - crocksdb_ffi::crocksdb_options_set_base_background_compactions(self.inner, n); - } - } - - pub fn get_base_background_compactions(&self) -> i32 { - unsafe { crocksdb_ffi::crocksdb_options_get_base_background_compactions(self.inner) } - } - pub fn set_max_background_flushes(&mut self, n: c_int) { unsafe { crocksdb_ffi::crocksdb_options_set_max_background_flushes(self.inner, n); diff --git a/tests/cases/test_rocksdb_options.rs b/tests/cases/test_rocksdb_options.rs index 2a60aba22..019a84c93 100644 --- a/tests/cases/test_rocksdb_options.rs +++ b/tests/cases/test_rocksdb_options.rs @@ -601,10 +601,8 @@ fn test_set_max_background_compactions_and_flushes() { opts.create_if_missing(true); opts.set_max_background_compactions(4); opts.set_max_background_flushes(1); - opts.set_base_background_compactions(2); assert_eq!(opts.get_max_background_compactions(), 4); assert_eq!(opts.get_max_background_flushes(), 1); - assert_eq!(opts.get_base_background_compactions(), 2); DB::open(opts, path.path().to_str().unwrap()).unwrap(); } @@ -914,8 +912,9 @@ fn test_compact_on_deletion() { let dels_trigger = 90; let mut opts = DBOptions::new(); - let cf_opts = ColumnFamilyOptions::new(); + let mut cf_opts = ColumnFamilyOptions::new(); opts.create_if_missing(true); + cf_opts.set_level_compaction_dynamic_level_bytes(false); cf_opts.set_compact_on_deletion(window_size, dels_trigger); let path = tempdir_with_prefix("_rust_rocksdb_compact_on_deletion_test"); From f4c57338209401de1944af7c7d63b7f1267b036a Mon Sep 17 00:00:00 2001 From: Bisheng Huang Date: Tue, 25 Jun 2024 15:23:13 +0800 Subject: [PATCH 2/3] [WIP] Expose WAL compression option Signed-off-by: Bisheng Huang --- .gitmodules | 8 ++++---- librocksdb_sys/crocksdb/c.cc | 8 ++++++++ librocksdb_sys/crocksdb/crocksdb/c.h | 5 +++++ librocksdb_sys/libtitan_sys/titan | 2 +- librocksdb_sys/rocksdb | 2 +- librocksdb_sys/src/lib.rs | 5 +++++ src/rocksdb_options.rs | 8 ++++++++ 7 files changed, 32 insertions(+), 6 deletions(-) diff --git a/.gitmodules b/.gitmodules index 39b057be9..4b0480476 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,9 +1,9 @@ [submodule "rocksdb"] path = librocksdb_sys/rocksdb - url = https://github.com/v01dstar/rocksdb.git - branch = 8.10-tikv + url = https://github.com/hbisheng/rocksdb.git + branch = fix-pri [submodule "titan"] path = librocksdb_sys/libtitan_sys/titan - url = https://github.com/v01dstar/titan.git - branch = upgrade-rocksdb + url = https://github.com/hbisheng/titan.git + branch = fix-pri diff --git a/librocksdb_sys/crocksdb/c.cc b/librocksdb_sys/crocksdb/c.cc index df49844a4..325f89121 100644 --- a/librocksdb_sys/crocksdb/c.cc +++ b/librocksdb_sys/crocksdb/c.cc @@ -5070,6 +5070,14 @@ void crocksdb_options_set_min_level_to_compress(crocksdb_options_t* opt, } } +uint32_t crocksdb_options_get_wal_compression(crocksdb_options_t* opt) { + return static_cast(opt->rep.wal_compression); +} + +void crocksdb_options_set_wal_compression(crocksdb_options_t* opt, int compression_type) { + opt->rep.wal_compression = static_cast(compression_type); +} + size_t crocksdb_livefiles_count(const crocksdb_livefiles_t* lf) { return static_cast(lf->rep.size()); } diff --git a/librocksdb_sys/crocksdb/crocksdb/c.h b/librocksdb_sys/crocksdb/crocksdb/c.h index 622585b95..7b03533ed 100644 --- a/librocksdb_sys/crocksdb/crocksdb/c.h +++ b/librocksdb_sys/crocksdb/crocksdb/c.h @@ -1345,6 +1345,11 @@ extern C_ROCKSDB_LIBRARY_API void crocksdb_options_set_plain_table_factory( extern C_ROCKSDB_LIBRARY_API void crocksdb_options_set_min_level_to_compress( crocksdb_options_t* opt, int level); +extern C_ROCKSDB_LIBRARY_API uint32_t crocksdb_options_get_wal_compression( + crocksdb_options_t* opt); +extern C_ROCKSDB_LIBRARY_API void crocksdb_options_set_wal_compression( + crocksdb_options_t* opt, int compression_type); + extern C_ROCKSDB_LIBRARY_API void crocksdb_options_set_memtable_huge_page_size( crocksdb_options_t*, size_t); diff --git a/librocksdb_sys/libtitan_sys/titan b/librocksdb_sys/libtitan_sys/titan index 0e453240c..8b9f815b6 160000 --- a/librocksdb_sys/libtitan_sys/titan +++ b/librocksdb_sys/libtitan_sys/titan @@ -1 +1 @@ -Subproject commit 0e453240c28632342bc46ce67a9df5c067764aef +Subproject commit 8b9f815b6003537a8b63ff1ca8de55f8fe4fecd2 diff --git a/librocksdb_sys/rocksdb b/librocksdb_sys/rocksdb index 620dcb341..428b792cf 160000 --- a/librocksdb_sys/rocksdb +++ b/librocksdb_sys/rocksdb @@ -1 +1 @@ -Subproject commit 620dcb341c33161cb3ef29e8b146620cfd467564 +Subproject commit 428b792cfcf04c50e63253b937da4cce266abbc8 diff --git a/librocksdb_sys/src/lib.rs b/librocksdb_sys/src/lib.rs index b5014b830..83b0252f5 100644 --- a/librocksdb_sys/src/lib.rs +++ b/librocksdb_sys/src/lib.rs @@ -847,6 +847,11 @@ extern "C" { zstd_max_train_bytes: c_int, parallel_threads: c_int, ); + pub fn crocksdb_options_set_wal_compression( + options: *mut Options, + compression_style_no: DBCompressionType, + ); + pub fn crocksdb_options_get_wal_compression(options: *mut Options) -> DBCompressionType; pub fn crocksdb_options_set_bottommost_compression_options( options: *mut Options, window_bits: c_int, diff --git a/src/rocksdb_options.rs b/src/rocksdb_options.rs index 3d40dfd79..46ca07f04 100644 --- a/src/rocksdb_options.rs +++ b/src/rocksdb_options.rs @@ -1117,6 +1117,14 @@ impl DBOptions { } } + pub fn set_wal_compression(&self, c: DBCompressionType) { + unsafe { crocksdb_ffi::crocksdb_options_set_wal_compression(self.inner, c) } + } + + pub fn get_wal_compression(&self) -> DBCompressionType { + unsafe { crocksdb_ffi::crocksdb_options_get_wal_compression(self.inner) } + } + pub fn set_delayed_write_rate(&mut self, rate: u64) { unsafe { crocksdb_ffi::crocksdb_options_set_delayed_write_rate(self.inner, rate); From 1958cea67570611349a985c128a8f8b7dec8983b Mon Sep 17 00:00:00 2001 From: Bisheng Huang Date: Thu, 22 Aug 2024 17:07:34 +0800 Subject: [PATCH 3/3] remove the get method Signed-off-by: Bisheng Huang --- librocksdb_sys/crocksdb/c.cc | 12 ++++-------- librocksdb_sys/crocksdb/crocksdb/c.h | 7 ++----- librocksdb_sys/src/lib.rs | 9 ++++----- src/rocksdb_options.rs | 12 ++++-------- 4 files changed, 14 insertions(+), 26 deletions(-) diff --git a/librocksdb_sys/crocksdb/c.cc b/librocksdb_sys/crocksdb/c.cc index 325f89121..4d36475db 100644 --- a/librocksdb_sys/crocksdb/c.cc +++ b/librocksdb_sys/crocksdb/c.cc @@ -3005,6 +3005,10 @@ void crocksdb_options_set_wal_size_limit_mb(crocksdb_options_t* opt, opt->rep.WAL_size_limit_MB = limit; } +void crocksdb_options_set_wal_compression(crocksdb_options_t* opt, int compression_type) { + opt->rep.wal_compression = static_cast(compression_type); +} + void crocksdb_options_set_manifest_preallocation_size(crocksdb_options_t* opt, size_t v) { opt->rep.manifest_preallocation_size = v; @@ -5070,14 +5074,6 @@ void crocksdb_options_set_min_level_to_compress(crocksdb_options_t* opt, } } -uint32_t crocksdb_options_get_wal_compression(crocksdb_options_t* opt) { - return static_cast(opt->rep.wal_compression); -} - -void crocksdb_options_set_wal_compression(crocksdb_options_t* opt, int compression_type) { - opt->rep.wal_compression = static_cast(compression_type); -} - size_t crocksdb_livefiles_count(const crocksdb_livefiles_t* lf) { return static_cast(lf->rep.size()); } diff --git a/librocksdb_sys/crocksdb/crocksdb/c.h b/librocksdb_sys/crocksdb/crocksdb/c.h index 7b03533ed..2a7de6bff 100644 --- a/librocksdb_sys/crocksdb/crocksdb/c.h +++ b/librocksdb_sys/crocksdb/crocksdb/c.h @@ -1270,6 +1270,8 @@ extern C_ROCKSDB_LIBRARY_API void crocksdb_options_set_wal_ttl_seconds( crocksdb_options_t*, uint64_t); extern C_ROCKSDB_LIBRARY_API void crocksdb_options_set_wal_size_limit_mb( crocksdb_options_t*, uint64_t); +extern C_ROCKSDB_LIBRARY_API void crocksdb_options_set_wal_compression( + crocksdb_options_t* opt, int compression_type); extern C_ROCKSDB_LIBRARY_API void crocksdb_options_set_manifest_preallocation_size(crocksdb_options_t*, size_t); extern C_ROCKSDB_LIBRARY_API void crocksdb_options_set_allow_mmap_reads( @@ -1345,11 +1347,6 @@ extern C_ROCKSDB_LIBRARY_API void crocksdb_options_set_plain_table_factory( extern C_ROCKSDB_LIBRARY_API void crocksdb_options_set_min_level_to_compress( crocksdb_options_t* opt, int level); -extern C_ROCKSDB_LIBRARY_API uint32_t crocksdb_options_get_wal_compression( - crocksdb_options_t* opt); -extern C_ROCKSDB_LIBRARY_API void crocksdb_options_set_wal_compression( - crocksdb_options_t* opt, int compression_type); - extern C_ROCKSDB_LIBRARY_API void crocksdb_options_set_memtable_huge_page_size( crocksdb_options_t*, size_t); diff --git a/librocksdb_sys/src/lib.rs b/librocksdb_sys/src/lib.rs index 83b0252f5..182051b83 100644 --- a/librocksdb_sys/src/lib.rs +++ b/librocksdb_sys/src/lib.rs @@ -847,11 +847,6 @@ extern "C" { zstd_max_train_bytes: c_int, parallel_threads: c_int, ); - pub fn crocksdb_options_set_wal_compression( - options: *mut Options, - compression_style_no: DBCompressionType, - ); - pub fn crocksdb_options_get_wal_compression(options: *mut Options) -> DBCompressionType; pub fn crocksdb_options_set_bottommost_compression_options( options: *mut Options, window_bits: c_int, @@ -936,6 +931,10 @@ extern "C" { pub fn crocksdb_options_set_wal_dir(options: *mut Options, path: *const c_char); pub fn crocksdb_options_set_wal_ttl_seconds(options: *mut Options, ttl: u64); pub fn crocksdb_options_set_wal_size_limit_mb(options: *mut Options, limit: u64); + pub fn crocksdb_options_set_wal_compression( + options: *mut Options, + compression_style_no: DBCompressionType, + ); pub fn crocksdb_options_set_use_direct_reads(options: *mut Options, v: bool); pub fn crocksdb_options_set_use_direct_io_for_flush_and_compaction( options: *mut Options, diff --git a/src/rocksdb_options.rs b/src/rocksdb_options.rs index 46ca07f04..c88275c10 100644 --- a/src/rocksdb_options.rs +++ b/src/rocksdb_options.rs @@ -1117,14 +1117,6 @@ impl DBOptions { } } - pub fn set_wal_compression(&self, c: DBCompressionType) { - unsafe { crocksdb_ffi::crocksdb_options_set_wal_compression(self.inner, c) } - } - - pub fn get_wal_compression(&self) -> DBCompressionType { - unsafe { crocksdb_ffi::crocksdb_options_get_wal_compression(self.inner) } - } - pub fn set_delayed_write_rate(&mut self, rate: u64) { unsafe { crocksdb_ffi::crocksdb_options_set_delayed_write_rate(self.inner, rate); @@ -1181,6 +1173,10 @@ impl DBOptions { } } + pub fn set_wal_compression(&self, c: DBCompressionType) { + unsafe { crocksdb_ffi::crocksdb_options_set_wal_compression(self.inner, c) } + } + pub fn set_max_log_file_size(&mut self, size: u64) { unsafe { crocksdb_ffi::crocksdb_options_set_max_log_file_size(self.inner, size as size_t);