From 43b43d9749ea14a33e90da1e66a25197bcd91450 Mon Sep 17 00:00:00 2001 From: Yeastplume Date: Tue, 6 Feb 2024 16:33:32 +0000 Subject: [PATCH] croaring-rs upgrade 0.5.2 -> 1.0.1 (#3779) * first pass compilation of croaring update * cargo.lock * add roaring arch flag into CI build scripts * revert CI to use windows 2019 image * add more debug * more debug info * update range arguments to bitmap remove_range function calls * remove unnecessary cast --- .github/workflows/cd.yaml | 4 +- .github/workflows/ci.yaml | 4 +- Cargo.lock | 216 +++------------------- chain/Cargo.toml | 2 +- chain/src/txhashset/bitmap_accumulator.rs | 4 +- chain/src/txhashset/txhashset.rs | 19 +- chain/tests/bitmap_segment.rs | 2 +- chain/tests/test_pibd_validation.rs | 2 +- core/Cargo.toml | 2 +- core/src/core/pmmr/segment.rs | 4 +- core/src/pow/cuckatoo.rs | 4 +- core/src/pow/lean.rs | 8 +- store/Cargo.toml | 2 +- store/src/leaf_set.rs | 22 +-- store/src/lib.rs | 2 +- store/src/pmmr.rs | 4 +- store/src/prune_list.rs | 17 +- store/tests/pmmr.rs | 24 +-- store/tests/segment.rs | 40 ++-- store/tests/test_bitmap.rs | 18 +- 20 files changed, 119 insertions(+), 281 deletions(-) diff --git a/.github/workflows/cd.yaml b/.github/workflows/cd.yaml index d753c9ad95..4f5c88126f 100644 --- a/.github/workflows/cd.yaml +++ b/.github/workflows/cd.yaml @@ -50,7 +50,9 @@ jobs: windows-release: name: Windows Release - runs-on: windows-latest + runs-on: windows-2019 + env: + ROARING_ARCH: x86-64-v2 steps: - name: Checkout uses: actions/checkout@v3 diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index d1ebb62c44..3e64c03f08 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -25,7 +25,9 @@ jobs: windows-tests: name: Windows Tests - runs-on: windows-latest + runs-on: windows-2019 + env: + ROARING_ARCH: x86-64-v2 steps: - name: Checkout uses: actions/checkout@v3 diff --git a/Cargo.lock b/Cargo.lock index 7c1d2a121d..7d16fe6f95 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -156,25 +156,6 @@ version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8c3c1a368f70d6cf7302d78f8f7093da241fb8e8807c05cc9e51a125895a6d5b" -[[package]] -name = "bindgen" -version = "0.59.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2bd2a9a458e8f4304c52c43ebb0cfbd520289f8379a52e329a38afda99bf8eb8" -dependencies = [ - "bitflags 1.3.2", - "cexpr", - "clang-sys", - "lazy_static", - "lazycell", - "peeking_take_while", - "proc-macro2 1.0.78", - "quote 1.0.35", - "regex", - "rustc-hash", - "shlex", -] - [[package]] name = "bit-vec" version = "0.6.3" @@ -274,15 +255,6 @@ dependencies = [ "libc", ] -[[package]] -name = "cexpr" -version = "0.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6fac387a98bb7c37292057cffc56d62ecb629900026402633ae9160df93a8766" -dependencies = [ - "nom", -] - [[package]] name = "cfg-if" version = "0.1.10" @@ -307,18 +279,7 @@ dependencies = [ "num-traits 0.2.17", "serde", "wasm-bindgen", - "windows-targets 0.52.0", -] - -[[package]] -name = "clang-sys" -version = "1.7.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "67523a3b4be3ce1989d607a828d036249522dd9c1c8de7f4dd2dae43a37369d1" -dependencies = [ - "glob", - "libc", - "libloading", + "windows-targets", ] [[package]] @@ -394,24 +355,21 @@ dependencies = [ [[package]] name = "croaring" -version = "0.5.2" +version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aff1eea8a79ffa2a743c1322d5c3853d45699b7842197160c7c32a18c32c1866" +checksum = "7266f0a7275b00ce4c4f4753e8c31afdefe93828101ece83a06e2ddab1dd1010" dependencies = [ "byteorder", "croaring-sys", - "libc", ] [[package]] name = "croaring-sys" -version = "0.5.2" +version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "21d77e33a1d5e04573f79846692e72f2e02e0fdd942b90f023c45f146d3447db" +checksum = "e47112498c394a7067949ebc07ef429b7384a413cf0efcf675846a47bcd307fb" dependencies = [ - "bindgen", "cc", - "libc", ] [[package]] @@ -465,7 +423,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b467862cc8610ca6fc9a1532d7777cee0804e678ab45410897b9396495994a0b" dependencies = [ "nix", - "windows-sys 0.52.0", + "windows-sys", ] [[package]] @@ -520,9 +478,9 @@ dependencies = [ [[package]] name = "darling" -version = "0.20.3" +version = "0.20.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0209d94da627ab5605dcccf08bb18afa5009cfbef48d8a8b7d7bdbc79be25c5e" +checksum = "da01daa5f6d41c91358398e8db4dde38e292378da1f28300b59ef4732b879454" dependencies = [ "darling_core", "darling_macro", @@ -530,9 +488,9 @@ dependencies = [ [[package]] name = "darling_core" -version = "0.20.3" +version = "0.20.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "177e3443818124b357d8e76f53be906d60937f0d3a90773a664fa63fa253e621" +checksum = "f44f6238b948a3c6c3073cdf53bb0c2d5e024ee27e0f35bfe9d556a12395808a" dependencies = [ "fnv", "ident_case", @@ -543,9 +501,9 @@ dependencies = [ [[package]] name = "darling_macro" -version = "0.20.3" +version = "0.20.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "836a9bbc7ad63342d6d6e7b815ccab164bc77a2d95d84bc3117a8c0d5c98e2d5" +checksum = "0d2d88bd93979b1feb760a6b5c531ac5ba06bd63e74894c377af02faee07b9cd" dependencies = [ "darling_core", "quote 1.0.35", @@ -692,7 +650,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a258e46cdc063eb8519c00b9fc845fc47bcfca4130e2f08e88665ceda8474245" dependencies = [ "libc", - "windows-sys 0.52.0", + "windows-sys", ] [[package]] @@ -710,7 +668,7 @@ dependencies = [ "cfg-if 1.0.0", "libc", "redox_syscall 0.4.1", - "windows-sys 0.52.0", + "windows-sys", ] [[package]] @@ -911,12 +869,6 @@ dependencies = [ "url", ] -[[package]] -name = "glob" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d2fabcfbdc87f4758337ca535fb41a6d701b65693ce38287d856d1674551ec9b" - [[package]] name = "grin" version = "5.3.0-alpha.1" @@ -1468,12 +1420,6 @@ version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" -[[package]] -name = "lazycell" -version = "1.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55" - [[package]] name = "libc" version = "0.2.152" @@ -1502,16 +1448,6 @@ dependencies = [ "libc", ] -[[package]] -name = "libloading" -version = "0.8.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c571b676ddfc9a8c12f1f3d3085a7b163966a8fd8098a90640953ce5f6170161" -dependencies = [ - "cfg-if 1.0.0", - "windows-sys 0.48.0", -] - [[package]] name = "libredox" version = "0.0.1" @@ -1639,12 +1575,6 @@ dependencies = [ "winapi 0.3.9", ] -[[package]] -name = "minimal-lexical" -version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" - [[package]] name = "miniz_oxide" version = "0.7.1" @@ -1756,16 +1686,6 @@ version = "0.1.14" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72ef4a56884ca558e5ddb05a1d1e7e1bfd9a68d9ed024c21704cc98872dae1bb" -[[package]] -name = "nom" -version = "7.1.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a" -dependencies = [ - "memchr", - "minimal-lexical", -] - [[package]] name = "num" version = "0.2.1" @@ -2035,12 +1955,6 @@ dependencies = [ "libc", ] -[[package]] -name = "peeking_take_while" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "19b17cddbe7ec3f8bc800887bab5e717348c95ea2ca0b1bf0837fb964dc67099" - [[package]] name = "percent-encoding" version = "2.3.1" @@ -2372,12 +2286,6 @@ version = "0.1.23" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d626bb9dae77e28219937af045c257c28bfd3f69333c512553507f5f9798cb76" -[[package]] -name = "rustc-hash" -version = "1.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2" - [[package]] name = "rustc-serialize" version = "0.3.25" @@ -2394,7 +2302,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys", - "windows-sys 0.52.0", + "windows-sys", ] [[package]] @@ -2443,7 +2351,7 @@ version = "0.1.23" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fbc91545643bcf3a0bbb6569265615222618bdf33ce4ffbbd13c4bbd4c093534" dependencies = [ - "windows-sys 0.52.0", + "windows-sys", ] [[package]] @@ -2567,12 +2475,6 @@ dependencies = [ "opaque-debug", ] -[[package]] -name = "shlex" -version = "1.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" - [[package]] name = "signal-hook" version = "0.3.17" @@ -2697,7 +2599,7 @@ dependencies = [ "fastrand", "redox_syscall 0.4.1", "rustix", - "windows-sys 0.52.0", + "windows-sys", ] [[package]] @@ -3203,16 +3105,7 @@ version = "0.52.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "33ab640c8d7e35bf8ba19b884ba838ceb4fba93a4e8c65a9059d08afcfc683d9" dependencies = [ - "windows-targets 0.52.0", -] - -[[package]] -name = "windows-sys" -version = "0.48.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "677d2418bec65e3338edb076e806bc1ec15693c5d0104683f2efe857f61056a9" -dependencies = [ - "windows-targets 0.48.5", + "windows-targets", ] [[package]] @@ -3221,22 +3114,7 @@ version = "0.52.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" dependencies = [ - "windows-targets 0.52.0", -] - -[[package]] -name = "windows-targets" -version = "0.48.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9a2fa6e2155d7247be68c096456083145c183cbbbc2764150dda45a87197940c" -dependencies = [ - "windows_aarch64_gnullvm 0.48.5", - "windows_aarch64_msvc 0.48.5", - "windows_i686_gnu 0.48.5", - "windows_i686_msvc 0.48.5", - "windows_x86_64_gnu 0.48.5", - "windows_x86_64_gnullvm 0.48.5", - "windows_x86_64_msvc 0.48.5", + "windows-targets", ] [[package]] @@ -3245,93 +3123,51 @@ version = "0.52.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8a18201040b24831fbb9e4eb208f8892e1f50a37feb53cc7ff887feb8f50e7cd" dependencies = [ - "windows_aarch64_gnullvm 0.52.0", - "windows_aarch64_msvc 0.52.0", - "windows_i686_gnu 0.52.0", - "windows_i686_msvc 0.52.0", - "windows_x86_64_gnu 0.52.0", - "windows_x86_64_gnullvm 0.52.0", - "windows_x86_64_msvc 0.52.0", + "windows_aarch64_gnullvm", + "windows_aarch64_msvc", + "windows_i686_gnu", + "windows_i686_msvc", + "windows_x86_64_gnu", + "windows_x86_64_gnullvm", + "windows_x86_64_msvc", ] -[[package]] -name = "windows_aarch64_gnullvm" -version = "0.48.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2b38e32f0abccf9987a4e3079dfb67dcd799fb61361e53e2882c3cbaf0d905d8" - [[package]] name = "windows_aarch64_gnullvm" version = "0.52.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cb7764e35d4db8a7921e09562a0304bf2f93e0a51bfccee0bd0bb0b666b015ea" -[[package]] -name = "windows_aarch64_msvc" -version = "0.48.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc35310971f3b2dbbf3f0690a219f40e2d9afcf64f9ab7cc1be722937c26b4bc" - [[package]] name = "windows_aarch64_msvc" version = "0.52.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bbaa0368d4f1d2aaefc55b6fcfee13f41544ddf36801e793edbbfd7d7df075ef" -[[package]] -name = "windows_i686_gnu" -version = "0.48.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a75915e7def60c94dcef72200b9a8e58e5091744960da64ec734a6c6e9b3743e" - [[package]] name = "windows_i686_gnu" version = "0.52.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a28637cb1fa3560a16915793afb20081aba2c92ee8af57b4d5f28e4b3e7df313" -[[package]] -name = "windows_i686_msvc" -version = "0.48.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f55c233f70c4b27f66c523580f78f1004e8b5a8b659e05a4eb49d4166cca406" - [[package]] name = "windows_i686_msvc" version = "0.52.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ffe5e8e31046ce6230cc7215707b816e339ff4d4d67c65dffa206fd0f7aa7b9a" -[[package]] -name = "windows_x86_64_gnu" -version = "0.48.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "53d40abd2583d23e4718fddf1ebec84dbff8381c07cae67ff7768bbf19c6718e" - [[package]] name = "windows_x86_64_gnu" version = "0.52.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3d6fa32db2bc4a2f5abeacf2b69f7992cd09dca97498da74a151a3132c26befd" -[[package]] -name = "windows_x86_64_gnullvm" -version = "0.48.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b7b52767868a23d5bab768e390dc5f5c55825b6d30b86c844ff2dc7414044cc" - [[package]] name = "windows_x86_64_gnullvm" version = "0.52.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1a657e1e9d3f514745a572a6846d3c7aa7dbe1658c056ed9c3344c4109a6949e" -[[package]] -name = "windows_x86_64_msvc" -version = "0.48.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538" - [[package]] name = "windows_x86_64_msvc" version = "0.52.0" diff --git a/chain/Cargo.toml b/chain/Cargo.toml index 7edbf3e845..e38f0f1d17 100644 --- a/chain/Cargo.toml +++ b/chain/Cargo.toml @@ -13,7 +13,7 @@ edition = "2018" bit-vec = "0.6" bitflags = "1" byteorder = "1" -croaring = "0.5.2" +croaring = "1.0.1" enum_primitive = "0.1" log = "0.4" serde = "1" diff --git a/chain/src/txhashset/bitmap_accumulator.rs b/chain/src/txhashset/bitmap_accumulator.rs index b0d8a0bc1b..cae45f7d8b 100644 --- a/chain/src/txhashset/bitmap_accumulator.rs +++ b/chain/src/txhashset/bitmap_accumulator.rs @@ -155,7 +155,7 @@ impl BitmapAccumulator { let last_pos = self.backend.size(); let mut pmmr = PMMR::at(&mut self.backend, last_pos); let rewind_pos = pmmr::insertion_to_pmmr_index(chunk_idx); - pmmr.rewind(rewind_pos, &Bitmap::create()) + pmmr.rewind(rewind_pos, &Bitmap::new()) .map_err(Error::Other)?; Ok(()) } @@ -193,7 +193,7 @@ impl BitmapAccumulator { /// Return a raw in-memory bitmap of this accumulator pub fn as_bitmap(&self) -> Result { - let mut bitmap = Bitmap::create(); + let mut bitmap = Bitmap::new(); for (chunk_index, chunk_pos) in self.backend.leaf_pos_iter().enumerate() { //TODO: Unwrap let chunk = self.backend.get_data(chunk_pos as u64).unwrap(); diff --git a/chain/src/txhashset/txhashset.rs b/chain/src/txhashset/txhashset.rs index db993bf802..a0d6bc64ca 100644 --- a/chain/src/txhashset/txhashset.rs +++ b/chain/src/txhashset/txhashset.rs @@ -1079,7 +1079,7 @@ impl<'a> HeaderExtension<'a> { let header_pos = 1 + pmmr::insertion_to_pmmr_index(header.height); self.pmmr - .rewind(header_pos, &Bitmap::create()) + .rewind(header_pos, &Bitmap::new()) .map_err(&Error::TxHashSetErr)?; // Update our head to reflect the header we rewound to. @@ -1178,7 +1178,7 @@ impl<'a> Extension<'a> { bitmap_cache: trees .bitmap_accumulator .as_bitmap() - .unwrap_or(Bitmap::create()), + .unwrap_or(Bitmap::new()), rollback: false, } } @@ -1302,10 +1302,7 @@ impl<'a> Extension<'a> { /// Sets the bitmap accumulator (as received during PIBD sync) pub fn set_bitmap_accumulator(&mut self, accumulator: BitmapAccumulator) { self.bitmap_accumulator = accumulator; - self.bitmap_cache = self - .bitmap_accumulator - .as_bitmap() - .unwrap_or(Bitmap::create()); + self.bitmap_cache = self.bitmap_accumulator.as_bitmap().unwrap_or(Bitmap::new()); } // Prune output and rangeproof PMMRs based on provided pos. @@ -1366,7 +1363,7 @@ impl<'a> Extension<'a> { /// Once the PIBD set is downloaded, we need to ensure that the respective leaf sets /// match the bitmap (particularly in the case of outputs being spent after a PIBD catch-up) pub fn update_leaf_sets(&mut self, bitmap: &Bitmap) -> Result<(), Error> { - let flipped = bitmap.flip(0..bitmap.maximum().unwrap() as u64 + 1); + let flipped = bitmap.flip(0u32..bitmap.maximum().unwrap() + 1); for spent_pmmr_index in flipped.iter() { let pos0 = pmmr::insertion_to_pmmr_index(spent_pmmr_index.into()); self.output_pmmr.remove_from_leaf_set(pos0); @@ -1420,7 +1417,7 @@ impl<'a> Extension<'a> { // All initial outputs are spent up to this hash, // Roll back the genesis output self.output_pmmr - .rewind(0, &Bitmap::create()) + .rewind(0, &Bitmap::new()) .map_err(&Error::TxHashSetErr)?; } self.output_pmmr @@ -1464,7 +1461,7 @@ impl<'a> Extension<'a> { // All initial outputs are spent up to this hash, // Roll back the genesis output self.rproof_pmmr - .rewind(0, &Bitmap::create()) + .rewind(0, &Bitmap::new()) .map_err(&Error::TxHashSetErr)?; } self.rproof_pmmr @@ -1715,7 +1712,7 @@ impl<'a> Extension<'a> { .rewind(output_pos, &bitmap) .map_err(&Error::TxHashSetErr)?; self.kernel_pmmr - .rewind(kernel_pos, &Bitmap::create()) + .rewind(kernel_pos, &Bitmap::new()) .map_err(&Error::TxHashSetErr)?; Ok(()) } @@ -2197,7 +2194,7 @@ fn input_pos_to_rewind( head_header: &BlockHeader, batch: &Batch<'_>, ) -> Result { - let mut bitmap = Bitmap::create(); + let mut bitmap = Bitmap::new(); let mut current = head_header.clone(); while current.height > block_header.height { if let Ok(block_bitmap) = batch.get_block_input_bitmap(¤t.hash()) { diff --git a/chain/tests/bitmap_segment.rs b/chain/tests/bitmap_segment.rs index c7fd0e714d..960625baf7 100644 --- a/chain/tests/bitmap_segment.rs +++ b/chain/tests/bitmap_segment.rs @@ -19,7 +19,7 @@ fn test_roundtrip(entries: usize) { }; let block = rng.gen_range(2, 64); - let mut bitmap = Bitmap::create(); + let mut bitmap = Bitmap::new(); let block_size = 1 << 16; let offset = (1 << identifier.height) * 1024 * identifier.idx + block_size * block; let mut count = 0; diff --git a/chain/tests/test_pibd_validation.rs b/chain/tests/test_pibd_validation.rs index 286b05601d..5f67b07a37 100644 --- a/chain/tests/test_pibd_validation.rs +++ b/chain/tests/test_pibd_validation.rs @@ -103,7 +103,7 @@ fn test_pibd_chain_validation_impl(is_test_chain: bool, src_root_dir: &str) { let mut bitmap_accumulator = BitmapAccumulator::new(); // Raw bitmap for validation - let mut bitmap = Bitmap::create(); + let mut bitmap = Bitmap::new(); let mut chunk_count = 0; for sid in identifier_iter { diff --git a/core/Cargo.toml b/core/Cargo.toml index d6a0ca0746..1c1444e38a 100644 --- a/core/Cargo.toml +++ b/core/Cargo.toml @@ -12,7 +12,7 @@ edition = "2018" [dependencies] blake2 = { package = "blake2-rfc", version = "0.2"} byteorder = "1" -croaring = "0.5.2" +croaring = "1.0.1" enum_primitive = "0.1" lazy_static = "1" lru-cache = "0.1" diff --git a/core/src/core/pmmr/segment.rs b/core/src/core/pmmr/segment.rs index 148a827732..8df8b8f989 100644 --- a/core/src/core/pmmr/segment.rs +++ b/core/src/core/pmmr/segment.rs @@ -508,8 +508,8 @@ where if let Some((p0, _)) = family_branch.next() { pos0 = p0; - let range = (pmmr::n_leaves(1 + pmmr::bintree_leftmost(p0)) - 1) - ..min(pmmr::n_leaves(1 + pmmr::bintree_rightmost(p0)), n_leaves); + let range = (pmmr::n_leaves(1 + pmmr::bintree_leftmost(p0)) - 1) as u32 + ..min(pmmr::n_leaves(1 + pmmr::bintree_rightmost(p0)), n_leaves) as u32; cardinality = bitmap.range_cardinality(range); } else { break; diff --git a/core/src/pow/cuckatoo.rs b/core/src/pow/cuckatoo.rs index eff3d3168b..013e64145c 100644 --- a/core/src/pow/cuckatoo.rs +++ b/core/src/pow/cuckatoo.rs @@ -56,7 +56,7 @@ impl Graph { proof_size, links: vec![], adj_list: vec![], - visited: Bitmap::create(), + visited: Bitmap::new(), solutions: vec![], nil: u64::max_value(), }) @@ -67,7 +67,7 @@ impl Graph { self.links = Vec::with_capacity(2 * self.max_nodes as usize); self.adj_list = vec![u64::max_value(); 2 * self.max_nodes as usize]; self.solutions = vec![Proof::zero(self.proof_size); 1]; - self.visited = Bitmap::create(); + self.visited = Bitmap::new(); Ok(()) } diff --git a/core/src/pow/lean.rs b/core/src/pow/lean.rs index 8d46efdae7..9a58315471 100644 --- a/core/src/pow/lean.rs +++ b/core/src/pow/lean.rs @@ -36,8 +36,8 @@ impl Lean { let params = CuckooParams::new(edge_bits, edge_bits, 42).unwrap(); // edge bitmap, before trimming all of them are on - let mut edges = Bitmap::create_with_capacity(params.num_edges as u32); - edges.flip_inplace(0..params.num_edges); + let mut edges = Bitmap::with_container_capacity(params.num_edges as u32); + edges.flip_inplace(0u32..params.num_edges as u32); Lean { params, edges } } @@ -65,7 +65,7 @@ impl Lean { fn count_and_kill(&mut self) { // on each side u or v of the bipartite graph for uorv in 0..2 { - let mut nodes = Bitmap::create(); + let mut nodes = Bitmap::new(); // increment count for each node for e in self.edges.iter() { let node = self.params.sipnode(e.into(), uorv).unwrap(); @@ -73,7 +73,7 @@ impl Lean { } // then kill edges with lone nodes (no neighbour at ^1) - let mut to_kill = Bitmap::create(); + let mut to_kill = Bitmap::new(); for e in self.edges.iter() { let node = self.params.sipnode(e.into(), uorv).unwrap(); if !nodes.contains((node ^ 1) as u32) { diff --git a/store/Cargo.toml b/store/Cargo.toml index c99ec69c3e..cc00d2cdcc 100644 --- a/store/Cargo.toml +++ b/store/Cargo.toml @@ -11,7 +11,7 @@ edition = "2018" [dependencies] byteorder = "1" -croaring = "0.5.2" +croaring = "1.0.1" libc = "0.2" lmdb-zero = "0.4.4" memmap = "0.7" diff --git a/store/src/leaf_set.rs b/store/src/leaf_set.rs index 0af75f519d..d0cc25531f 100644 --- a/store/src/leaf_set.rs +++ b/store/src/leaf_set.rs @@ -17,7 +17,7 @@ use std::path::{Path, PathBuf}; -use croaring::Bitmap; +use croaring::{Bitmap, Portable}; use crate::grin_core::core::hash::Hashed; use crate::grin_core::core::pmmr; @@ -44,14 +44,14 @@ impl LeafSet { let bitmap = if file_path.exists() { read_bitmap(&file_path)? } else { - Bitmap::create() + Bitmap::new() }; if !bitmap.is_empty() { debug!( "bitmap {} pos ({} bytes)", bitmap.cardinality(), - bitmap.get_serialized_size_in_bytes(), + bitmap.get_serialized_size_in_bytes::(), ); } @@ -114,8 +114,8 @@ impl LeafSet { // First remove pos from leaf_set that were // added after the point we are rewinding to. - let to_remove = ((cutoff_pos + 1) as u32)..bitmap.maximum().unwrap_or(0); - bitmap.remove_range_closed(to_remove); + let to_remove = ((cutoff_pos + 1) as u32)..=bitmap.maximum().unwrap_or(0); + bitmap.remove_range(to_remove); // Then add back output pos to the leaf_set // that were removed. @@ -123,7 +123,7 @@ impl LeafSet { // Invert bitmap for the leaf pos and return the resulting bitmap. bitmap - .flip(1..(cutoff_pos + 1)) + .flip(1u32..(cutoff_pos + 1) as u32) .and(&self.unpruned_pre_cutoff(cutoff_pos, prune_list)) } @@ -133,8 +133,8 @@ impl LeafSet { pub fn rewind(&mut self, cutoff_pos: u64, rewind_rm_pos: &Bitmap) { // First remove pos from leaf_set that were // added after the point we are rewinding to. - let to_remove = ((cutoff_pos + 1) as u32)..self.bitmap.maximum().unwrap_or(0); - self.bitmap.remove_range_closed(to_remove); + let to_remove = ((cutoff_pos + 1) as u32)..=self.bitmap.maximum().unwrap_or(0); + self.bitmap.remove_range(to_remove); // Then add back output pos to the leaf_set // that were removed. @@ -160,7 +160,7 @@ impl LeafSet { let cp_path = format!("{}.{}", self.path.to_str().unwrap(), header.hash()); let mut file = BufWriter::new(File::create(cp_path)?); - file.write_all(&cp_bitmap.serialize())?; + file.write_all(&cp_bitmap.serialize::())?; file.flush()?; Ok(()) } @@ -172,7 +172,7 @@ impl LeafSet { // Write the updated bitmap file to disk. save_via_temp_file(&self.path, ".tmp", |file| { - file.write_all(&self.bitmap.serialize()) + file.write_all(&self.bitmap.serialize::()) })?; // Make sure our backup in memory is up to date. @@ -198,7 +198,7 @@ impl LeafSet { /// Number of positions up to index n in the leaf set pub fn n_unpruned_leaves_to_index(&self, to_index: u64) -> u64 { - self.bitmap.range_cardinality(0..to_index) + self.bitmap.range_cardinality(0u32..to_index as u32) } /// Is the leaf_set empty. diff --git a/store/src/lib.rs b/store/src/lib.rs index d9ec1d419c..215b43f70f 100644 --- a/store/src/lib.rs +++ b/store/src/lib.rs @@ -118,5 +118,5 @@ pub fn read_bitmap>(file_path: P) -> io::Result { let f_md = bitmap_file.metadata()?; let mut buffer = Vec::with_capacity(f_md.len() as usize); bitmap_file.read_to_end(&mut buffer)?; - Ok(Bitmap::deserialize(&buffer)) + Ok(Bitmap::deserialize::(&buffer)) } diff --git a/store/src/pmmr.rs b/store/src/pmmr.rs index 8f83e1cb34..8a64d6822d 100644 --- a/store/src/pmmr.rs +++ b/store/src/pmmr.rs @@ -238,7 +238,7 @@ impl Backend for PMMRBackend { } fn reset_prune_list(&mut self) { - let bitmap = Bitmap::create(); + let bitmap = Bitmap::new(); self.prune_list = PruneList::new(Some(self.data_dir.join(PMMR_PRUN_FILE)), bitmap); if let Err(e) = self.prune_list.flush() { error!("Flushing reset prune list: {}", e); @@ -477,7 +477,7 @@ impl PMMRBackend { } fn pos_to_rm(&self, cutoff_pos: u64, rewind_rm_pos: &Bitmap) -> (Bitmap, Bitmap) { - let mut expanded = Bitmap::create(); + let mut expanded = Bitmap::new(); let leaf_pos_to_rm = self.leaf_set diff --git a/store/src/prune_list.rs b/store/src/prune_list.rs index a275322edd..2becf85f65 100644 --- a/store/src/prune_list.rs +++ b/store/src/prune_list.rs @@ -27,7 +27,7 @@ use std::{ ops::Range, }; -use croaring::Bitmap; +use croaring::{Bitmap, Portable}; use grin_core::core::pmmr; use crate::grin_core::core::pmmr::{bintree_leftmost, bintree_postorder_height, family}; @@ -61,7 +61,7 @@ impl PruneList { assert!(!bitmap.contains(0)); let mut prune_list = PruneList { path, - bitmap: Bitmap::create(), + bitmap: Bitmap::new(), shift_cache: vec![], leaf_shift_cache: vec![], }; @@ -76,7 +76,7 @@ impl PruneList { /// Instatiate a new empty prune list. pub fn empty() -> PruneList { - PruneList::new(None, Bitmap::create()) + PruneList::new(None, Bitmap::new()) } /// Open an existing prune_list or create a new one. @@ -86,7 +86,7 @@ impl PruneList { let bitmap = if file_path.exists() { read_bitmap(&file_path)? } else { - Bitmap::create() + Bitmap::new() }; assert!(!bitmap.contains(0)); @@ -99,7 +99,7 @@ impl PruneList { debug!( "bitmap {} pos ({} bytes), shift_cache {}, leaf_shift_cache {}", prune_list.bitmap.cardinality(), - prune_list.bitmap.get_serialized_size_in_bytes(), + prune_list.bitmap.get_serialized_size_in_bytes::(), prune_list.shift_cache.len(), prune_list.leaf_shift_cache.len(), ); @@ -122,7 +122,7 @@ impl PruneList { // Write the updated bitmap file to disk. if let Some(ref path) = self.path { save_via_temp_file(path, ".tmp", |file| { - file.write_all(&self.bitmap.serialize()) + file.write_all(&self.bitmap.serialize::()) })?; } @@ -258,14 +258,15 @@ impl PruneList { } // Note: We will treat this as a "closed range" below (croaring api weirdness). - let cleanup_pos1 = (lc0 + 1)..size; + // Note: After croaring upgrade to 1.0.2 we provide an inclusive range directly + let cleanup_pos1 = (lc0 + 1)..=size; // Find point where we can truncate based on bitmap "rank" (index) of pos to the left of subtree. let idx = self.bitmap.rank(lc0); self.shift_cache.truncate(idx as usize); self.leaf_shift_cache.truncate(idx as usize); - self.bitmap.remove_range_closed(cleanup_pos1) + self.bitmap.remove_range(cleanup_pos1) } /// Push the node at the provided position in the prune list. diff --git a/store/tests/pmmr.rs b/store/tests/pmmr.rs index 411d3bb2ab..935b39a6b4 100644 --- a/store/tests/pmmr.rs +++ b/store/tests/pmmr.rs @@ -192,7 +192,7 @@ fn pmmr_compact_leaf_sibling() { assert_eq!(backend.get_from_file(0).unwrap(), pos_0_hash); // aggressively compact the PMMR files - backend.check_compact(1, &Bitmap::create()).unwrap(); + backend.check_compact(1, &Bitmap::new()).unwrap(); // check pos 0, 1, 2 are in the state we expect after compacting { @@ -252,7 +252,7 @@ fn pmmr_prune_compact() { } // compact - backend.check_compact(2, &Bitmap::create()).unwrap(); + backend.check_compact(2, &Bitmap::new()).unwrap(); // recheck the root and stored data { @@ -302,7 +302,7 @@ fn pmmr_reload() { assert_eq!(backend.unpruned_size(), mmr_size); // now check and compact the backend - backend.check_compact(1, &Bitmap::create()).unwrap(); + backend.check_compact(1, &Bitmap::new()).unwrap(); assert_eq!(backend.unpruned_size(), mmr_size); backend.sync().unwrap(); assert_eq!(backend.unpruned_size(), mmr_size); @@ -316,7 +316,7 @@ fn pmmr_reload() { backend.sync().unwrap(); assert_eq!(backend.unpruned_size(), mmr_size); - backend.check_compact(4, &Bitmap::create()).unwrap(); + backend.check_compact(4, &Bitmap::new()).unwrap(); backend.sync().unwrap(); assert_eq!(backend.unpruned_size(), mmr_size); @@ -414,7 +414,7 @@ fn pmmr_rewind() { backend.sync().unwrap(); // and compact the MMR to remove the pruned elements - backend.check_compact(6, &Bitmap::create()).unwrap(); + backend.check_compact(6, &Bitmap::new()).unwrap(); backend.sync().unwrap(); println!("root1 {:?}, root2 {:?}, root3 {:?}", root1, root2, root3); @@ -459,7 +459,7 @@ fn pmmr_rewind() { { let mut pmmr: PMMR<'_, TestElem, _> = PMMR::at(&mut backend, 10); - pmmr.rewind(5, &Bitmap::create()).unwrap(); + pmmr.rewind(5, &Bitmap::new()).unwrap(); assert_eq!(pmmr.root().unwrap(), root1); } backend.sync().unwrap(); @@ -507,7 +507,7 @@ fn pmmr_compact_single_leaves() { backend.sync().unwrap(); // compact - backend.check_compact(2, &Bitmap::create()).unwrap(); + backend.check_compact(2, &Bitmap::new()).unwrap(); { let mut pmmr: PMMR<'_, TestElem, _> = PMMR::at(&mut backend, mmr_size); @@ -518,7 +518,7 @@ fn pmmr_compact_single_leaves() { backend.sync().unwrap(); // compact - backend.check_compact(2, &Bitmap::create()).unwrap(); + backend.check_compact(2, &Bitmap::new()).unwrap(); } teardown(data_dir); @@ -551,7 +551,7 @@ fn pmmr_compact_entire_peak() { backend.sync().unwrap(); // compact - backend.check_compact(2, &Bitmap::create()).unwrap(); + backend.check_compact(2, &Bitmap::new()).unwrap(); // now check we have pruned up to and including the peak at pos 7 // hash still available in underlying hash file @@ -702,7 +702,7 @@ fn pmmr_compact_horizon() { } // compact some more - backend.check_compact(9, &Bitmap::create()).unwrap(); + backend.check_compact(9, &Bitmap::new()).unwrap(); } // recheck stored data @@ -773,7 +773,7 @@ fn compact_twice() { } // compact - backend.check_compact(2, &Bitmap::create()).unwrap(); + backend.check_compact(2, &Bitmap::new()).unwrap(); // recheck the root and stored data { @@ -800,7 +800,7 @@ fn compact_twice() { } // compact - backend.check_compact(2, &Bitmap::create()).unwrap(); + backend.check_compact(2, &Bitmap::new()).unwrap(); // recheck the root and stored data { diff --git a/store/tests/segment.rs b/store/tests/segment.rs index c795b250b5..a24c1b8c5c 100644 --- a/store/tests/segment.rs +++ b/store/tests/segment.rs @@ -47,8 +47,8 @@ fn prunable_mmr() { let last_pos = mmr.unpruned_size(); let root = mmr.root().unwrap(); - let mut bitmap = Bitmap::create(); - bitmap.add_range(0..n_leaves as u64); + let mut bitmap = Bitmap::new(); + bitmap.add_range(0..n_leaves); let id = SegmentIdentifier { height: 3, idx: 1 }; @@ -65,7 +65,7 @@ fn prunable_mmr() { let mut mmr = PMMR::at(&mut ba, last_pos); prune(&mut mmr, &mut bitmap, &[8, 9, 13]); ba.sync().unwrap(); - ba.check_compact(last_pos, &Bitmap::create()).unwrap(); + ba.check_compact(last_pos, &Bitmap::new()).unwrap(); ba.sync().unwrap(); // Validate @@ -81,7 +81,7 @@ fn prunable_mmr() { let mut mmr = PMMR::at(&mut ba, last_pos); prune(&mut mmr, &mut bitmap, &[10, 11]); ba.sync().unwrap(); - ba.check_compact(last_pos, &Bitmap::create()).unwrap(); + ba.check_compact(last_pos, &Bitmap::new()).unwrap(); ba.sync().unwrap(); // Validate @@ -97,7 +97,7 @@ fn prunable_mmr() { let mut mmr = PMMR::at(&mut ba, last_pos); prune(&mut mmr, &mut bitmap, &[14, 15]); ba.sync().unwrap(); - ba.check_compact(last_pos, &Bitmap::create()).unwrap(); + ba.check_compact(last_pos, &Bitmap::new()).unwrap(); ba.sync().unwrap(); // Validate @@ -113,7 +113,7 @@ fn prunable_mmr() { let mut mmr = PMMR::at(&mut ba, last_pos); prune(&mut mmr, &mut bitmap, &[12]); ba.sync().unwrap(); - ba.check_compact(last_pos, &Bitmap::create()).unwrap(); + ba.check_compact(last_pos, &Bitmap::new()).unwrap(); ba.sync().unwrap(); let mmr = ReadonlyPMMR::at(&mut ba, last_pos); @@ -130,7 +130,7 @@ fn prunable_mmr() { let mut mmr = PMMR::at(&mut ba, last_pos); prune(&mut mmr, &mut bitmap, &[76, 77]); ba.sync().unwrap(); - ba.check_compact(last_pos, &Bitmap::create()).unwrap(); + ba.check_compact(last_pos, &Bitmap::new()).unwrap(); ba.sync().unwrap(); // Validate @@ -142,7 +142,7 @@ fn prunable_mmr() { let mut mmr = PMMR::at(&mut ba, last_pos); prune(&mut mmr, &mut bitmap, &[78]); ba.sync().unwrap(); - ba.check_compact(last_pos, &Bitmap::create()).unwrap(); + ba.check_compact(last_pos, &Bitmap::new()).unwrap(); ba.sync().unwrap(); // Validate @@ -173,13 +173,13 @@ fn pruned_segment() { let last_pos = mmr.unpruned_size(); let root = mmr.root().unwrap(); - let mut bitmap = Bitmap::create(); - bitmap.add_range(0..n_leaves as u64); + let mut bitmap = Bitmap::new(); + bitmap.add_range(0..n_leaves); // Prune all leaves of segment 1 prune(&mut mmr, &mut bitmap, &[4, 5, 6, 7]); ba.sync().unwrap(); - ba.check_compact(last_pos, &Bitmap::create()).unwrap(); + ba.check_compact(last_pos, &Bitmap::new()).unwrap(); ba.sync().unwrap(); // Validate the empty segment 1 @@ -201,7 +201,7 @@ fn pruned_segment() { let mut mmr = PMMR::at(&mut ba, last_pos); prune(&mut mmr, &mut bitmap, &[0, 1, 2, 3]); ba.sync().unwrap(); - ba.check_compact(last_pos, &Bitmap::create()).unwrap(); + ba.check_compact(last_pos, &Bitmap::new()).unwrap(); ba.sync().unwrap(); // Validate the empty segment 1 again @@ -223,7 +223,7 @@ fn pruned_segment() { let mut mmr = PMMR::at(&mut ba, last_pos); prune(&mut mmr, &mut bitmap, &[8, 9, 10, 11, 12, 13, 14, 15]); ba.sync().unwrap(); - ba.check_compact(last_pos, &Bitmap::create()).unwrap(); + ba.check_compact(last_pos, &Bitmap::new()).unwrap(); ba.sync().unwrap(); // Validate the empty segment 1 again @@ -246,7 +246,7 @@ fn pruned_segment() { for i in 16..n_leaves { mmr.push(&TestElem([i / 7, i / 5, i / 3, i])).unwrap(); } - bitmap.add_range(16..n_leaves as u64); + bitmap.add_range(16..n_leaves); let last_pos = mmr.unpruned_size(); let root = mmr.root().unwrap(); @@ -254,7 +254,7 @@ fn pruned_segment() { // The root of this segment is a direct peak of the full MMR prune(&mut mmr, &mut bitmap, &[16, 17, 18, 19]); ba.sync().unwrap(); - ba.check_compact(last_pos, &Bitmap::create()).unwrap(); + ba.check_compact(last_pos, &Bitmap::new()).unwrap(); ba.sync().unwrap(); // Validate segment 4 @@ -292,7 +292,7 @@ fn pruned_segment() { let mut mmr = PMMR::at(&mut ba, last_pos); prune(&mut mmr, &mut bitmap, &[22]); ba.sync().unwrap(); - ba.check_compact(last_pos, &Bitmap::create()).unwrap(); + ba.check_compact(last_pos, &Bitmap::new()).unwrap(); ba.sync().unwrap(); // Segment 5 should be unchanged @@ -305,7 +305,7 @@ fn pruned_segment() { let mut mmr = PMMR::at(&mut ba, last_pos); prune(&mut mmr, &mut bitmap, &[20, 21]); ba.sync().unwrap(); - ba.check_compact(last_pos, &Bitmap::create()).unwrap(); + ba.check_compact(last_pos, &Bitmap::new()).unwrap(); ba.sync().unwrap(); // Validate segment 5 again @@ -343,13 +343,13 @@ fn ser_round_trip() { for i in 0..n_leaves { mmr.push(&TestElem([i / 7, i / 5, i / 3, i])).unwrap(); } - let mut bitmap = Bitmap::create(); - bitmap.add_range(0..n_leaves as u64); + let mut bitmap = Bitmap::new(); + bitmap.add_range(0..n_leaves); let last_pos = mmr.unpruned_size(); prune(&mut mmr, &mut bitmap, &[0, 1]); ba.sync().unwrap(); - ba.check_compact(last_pos, &Bitmap::create()).unwrap(); + ba.check_compact(last_pos, &Bitmap::new()).unwrap(); ba.sync().unwrap(); let mmr = ReadonlyPMMR::at(&ba, last_pos); diff --git a/store/tests/test_bitmap.rs b/store/tests/test_bitmap.rs index a8b1388d12..e3d9d43b98 100644 --- a/store/tests/test_bitmap.rs +++ b/store/tests/test_bitmap.rs @@ -15,7 +15,7 @@ use rand; use chrono::prelude::Utc; -use croaring::Bitmap; +use croaring::{Bitmap, Portable}; use rand::Rng; // We can use "andnot" to rewind easily by passing in a "bitmask" of @@ -54,7 +54,7 @@ fn test_flip_bitmap() { #[test] fn test_a_small_bitmap() { let bitmap: Bitmap = vec![1, 99, 1_000].into_iter().collect(); - let serialized_buffer = bitmap.serialize(); + let serialized_buffer = bitmap.serialize::(); // we can store 3 pos in a roaring bitmap in 22 bytes // this is compared to storing them as a vec of u32 values which would be 4 * 3 @@ -65,18 +65,18 @@ fn test_a_small_bitmap() { #[test] fn test_1000_inputs() { let mut rng = rand::thread_rng(); - let mut bitmap = Bitmap::create(); + let mut bitmap = Bitmap::new(); for _ in 1..1_000 { let n = rng.gen_range(0, 1_000_000); bitmap.add(n); } - let serialized_buffer = bitmap.serialize(); + let serialized_buffer = bitmap.serialize::(); println!( "bitmap with 1,000 (out of 1,000,000) values in it: {}", serialized_buffer.len() ); bitmap.run_optimize(); - let serialized_buffer = bitmap.serialize(); + let serialized_buffer = bitmap.serialize::(); println!( "bitmap with 1,000 (out of 1,000,000) values in it (optimized): {}", serialized_buffer.len() @@ -86,7 +86,7 @@ fn test_1000_inputs() { #[test] fn test_a_big_bitmap() { let mut bitmap: Bitmap = (1..1_000_000).collect(); - let serialized_buffer = bitmap.serialize(); + let serialized_buffer = bitmap.serialize::(); // we can also store 1,000,000 pos in 131,208 bytes // a vec of u32s here would be 4,000,000 bytes @@ -94,7 +94,7 @@ fn test_a_big_bitmap() { // but note we can optimize this heavily to get down to 230 bytes... assert!(bitmap.run_optimize()); - let serialized_buffer = bitmap.serialize(); + let serialized_buffer = bitmap.serialize::(); assert_eq!(serialized_buffer.len(), 230); } @@ -110,7 +110,7 @@ fn bench_fast_or() { let mut rng = rand::thread_rng(); let mut bitmaps = vec![]; for _ in 0..bitmaps_number { - let mut bitmap = Bitmap::create(); + let mut bitmap = Bitmap::new(); for _ in 0..size_of_each_bitmap { let n = rng.gen_range(0, 1_000_000); bitmap.add(n); @@ -121,7 +121,7 @@ fn bench_fast_or() { }; let mut bitmaps = init_bitmaps(); - let mut bitmap = Bitmap::create(); + let mut bitmap = Bitmap::new(); let start = Utc::now().timestamp_nanos(); for _ in 0..bitmaps_number { bitmap.or_inplace(&bitmaps.pop().unwrap());