Upgrade RocksDB JNI to 10.10.1.1 while preserving 7.9.2 index compatibility

bookkeeper-dist/src/main/resources/LICENSE-all.bin.txt

@@ -277,7 +277,7 @@ Apache Software License, Version 2.
 - lib/org.eclipse.jetty.ee8-jetty-ee8-security-12.1.10.jar [22]
 - lib/org.eclipse.jetty.ee8-jetty-ee8-servlet-12.1.10.jar [22]
 - lib/org.eclipse.jetty.toolchain-jetty-servlet-api-4.0.9.jar [22]
-- lib/org.rocksdb-rocksdbjni-9.9.3.jar [23]
+- lib/org.rocksdb-rocksdbjni-10.10.1.1.jar [23]
 - lib/com.beust-jcommander-1.82.jar [24]
 - lib/org.apache.datasketches-datasketches-java-7.0.1.jar [25]
 - lib/org.apache.datasketches-datasketches-memory-4.1.0.jar [27]
@@ -364,7 +364,7 @@ Apache Software License, Version 2.
 [20] Source available at https://github.com/apache/commons-lang/tree/LANG_3_6
 [21] Source available at https://github.com/apache/zookeeper/tree/release-3.8.0
 [22] Source available at https://github.com/jetty/jetty.project/tree/jetty-12.1.10
-[23] Source available at https://github.com/facebook/rocksdb/tree/v9.9.3
+[23] Source available at https://github.com/facebook/rocksdb/tree/v10.10.1
 [24] Source available at https://github.com/cbeust/jcommander/tree/1.82
 [25] Source available at https://github.com/apache/datasketches-java/tree/7.0.1
 [26] Source available at https://github.com/yawkat/lz4-java/tree/v1.10.2
@@ -624,7 +624,7 @@ This private header is also used by Apple's open source
     * http://www.opensource.apple.com/source/configd/configd-453.19/dnsinfo/dnsinfo.h
 
 ------------------------------------------------------------------------------------
-lib/org.rocksdb-rocksdbjni-9.9.3.jar is derived from leveldb, which is under the following license.
+lib/org.rocksdb-rocksdbjni-10.10.1.1.jar is derived from leveldb, which is under the following license.
 
 Copyright (c) 2011 The LevelDB Authors. All rights reserved.
 

bookkeeper-dist/src/main/resources/LICENSE-server.bin.txt

@@ -277,7 +277,7 @@ Apache Software License, Version 2.
 - lib/org.eclipse.jetty.ee8-jetty-ee8-security-12.1.10.jar [22]
 - lib/org.eclipse.jetty.ee8-jetty-ee8-servlet-12.1.10.jar [22]
 - lib/org.eclipse.jetty.toolchain-jetty-servlet-api-4.0.9.jar [22]
-- lib/org.rocksdb-rocksdbjni-9.9.3.jar [23]
+- lib/org.rocksdb-rocksdbjni-10.10.1.1.jar [23]
 - lib/com.beust-jcommander-1.82.jar [24]
 - lib/org.apache.datasketches-datasketches-java-7.0.1.jar [25]
 - lib/org.apache.datasketches-datasketches-memory-4.1.0.jar [27]
@@ -360,7 +360,7 @@ Apache Software License, Version 2.
 [20] Source available at https://github.com/apache/commons-lang/tree/LANG_3_6
 [21] Source available at https://github.com/apache/zookeeper/tree/release-3.8.0
 [22] Source available at https://github.com/jetty/jetty.project/tree/jetty-12.1.10
-[23] Source available at https://github.com/facebook/rocksdb/tree/v9.9.3
+[23] Source available at https://github.com/facebook/rocksdb/tree/v10.10.1
 [24] Source available at https://github.com/cbeust/jcommander/tree/1.82
 [25] Source available at https://github.com/apache/datasketches-java/tree/7.0.1
 [26] Source available at https://github.com/yawkat/lz4-java/tree/v1.10.2
@@ -620,7 +620,7 @@ This private header is also used by Apple's open source
     * http://www.opensource.apple.com/source/configd/configd-453.19/dnsinfo/dnsinfo.h
 
 ------------------------------------------------------------------------------------
-lib/org.rocksdb-rocksdbjni-9.9.3.jar is derived from leveldb, which is under the following license.
+lib/org.rocksdb-rocksdbjni-10.10.1.1.jar is derived from leveldb, which is under the following license.
 
 Copyright (c) 2011 The LevelDB Authors. All rights reserved.
 

bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/storage/ldb/KeyValueStorageRocksDBTest.java

@@ -18,6 +18,7 @@
  */
 package org.apache.bookkeeper.bookie.storage.ldb;
 
+import static org.junit.Assert.assertArrayEquals;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertNotNull;
 import static org.junit.Assert.assertNull;
@@ -27,11 +28,18 @@
 import java.io.File;
 import java.io.IOException;
 import java.net.URL;
+import java.nio.ByteBuffer;
+import java.nio.ByteOrder;
 import java.nio.file.Files;
+import java.nio.file.Path;
 import java.nio.file.Paths;
 import java.util.List;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
 import org.apache.bookkeeper.conf.ServerConfiguration;
+import org.junit.Rule;
 import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
 import org.rocksdb.BlockBasedTableConfig;
 import org.rocksdb.ChecksumType;
 import org.rocksdb.ColumnFamilyDescriptor;
@@ -42,6 +50,53 @@
 
 public class KeyValueStorageRocksDBTest {
 
+    @Rule
+    public final TemporaryFolder temporaryFolder = new TemporaryFolder();
+
+    @Test
+    public void testCompatibleSstFormatWithAndWithoutConfigurationFiles() throws Exception {
+        for (boolean useConfigurationFile : new boolean[]{false, true}) {
+            for (KeyValueStorageFactory.DbConfigType type : KeyValueStorageFactory.DbConfigType.values()) {
+                File directory = temporaryFolder.newFolder();
+                ServerConfiguration configuration = new ServerConfiguration();
+                // Explicit nonexistent paths ensure that the fallback does not load local operator settings.
+                String defaultConf = new File(directory, "default.conf").toString();
+                String ledgerConf = new File(directory, "ledger.conf").toString();
+                String entryConf = new File(directory, "entry.conf").toString();
+                if (useConfigurationFile) {
+                    defaultConf = getClass().getClassLoader().getResource("conf/default_rocksdb.conf").getPath();
+                    ledgerConf = getClass().getClassLoader().getResource("conf/ledger_metadata_rocksdb.conf").getPath();
+                    entryConf = getClass().getClassLoader().getResource("conf/entry_location_rocksdb.conf").getPath();
+                }
+                configuration.setDefaultRocksDBConf(defaultConf);
+                configuration.setLedgerMetadataRocksdbConf(ledgerConf);
+                configuration.setEntryLocationRocksdbConf(entryConf);
+                byte[] key = new byte[]{1};
+                byte[] value = new byte[]{2};
+                try (KeyValueStorageRocksDB storage = new KeyValueStorageRocksDB(
+                        directory.toString(), "db", type, configuration)) {
+                    storage.put(key, value);
+                    storage.sync();
+                    storage.compact();
+                    assertArrayEquals(value, storage.get(key));
+                    List<Path> ssts;
+                    try (Stream<Path> files = Files.list(directory.toPath().resolve("db"))) {
+                        ssts = files.filter(path -> path.toString().endsWith(".sst")).collect(Collectors.toList());
+                    }
+                    assertTrue("Compaction must produce an SST", !ssts.isEmpty());
+                    for (Path sst : ssts) {
+                        byte[] bytes = Files.readAllBytes(sst);
+                        // The footer stores a little-endian format version before the 8-byte magic number.
+                        // TableProperties.getFormatVersion() is not the block-based table footer version.
+                        int format = ByteBuffer.wrap(bytes, bytes.length - 12, 4)
+                                .order(ByteOrder.LITTLE_ENDIAN).getInt();
+                        assertEquals("SST must be readable by RocksDB 7.9.2: " + sst, 5, format);
+                    }
+                }
+            }
+        }
+    }
+
     @Test
     public void testRocksDBInitiateWithBookieConfiguration() throws Exception {
         ServerConfiguration configuration = new ServerConfiguration();

bookkeeper-server/src/test/resources/conf/default_rocksdb.conf

@@ -17,6 +17,13 @@
 # */
 
 [DBOptions]
+ # Keep WAL records readable by RocksDB 7.9.2 (BookKeeper 4.17.0 / Pulsar 4.0.x).
+ # Enabling track_and_verify_wals adds predecessor records (since 9.11) that
+ # 7.9.2 cannot replay. Enable only after the downgrade window has closed to
+ # detect missing/truncated WALs during recovery. This option is unknown to
+ # 7.9.2: use that release's configuration files when downgrading.
+ # https://github.com/facebook/rocksdb/blob/v10.10.1/include/rocksdb/options.h
+ track_and_verify_wals=false
  # set by jni: options.setCreateIfMissing
  create_if_missing=true
  # set by jni: options.setInfoLogLevel
@@ -30,7 +37,14 @@
  #no default setting in CFOptions
 
 [TableOptions/BlockBasedTable "default"]
- # set by jni: tableOptions.setFormatVersion
+ # Keep newly flushed/compacted SSTs readable by RocksDB 7.9.2.
+ # Format 5 supports modern Bloom filters (readable since RocksDB 6.6).
+ # After the downgrade window closes, format 6 (requires >= 8.6) improves
+ # detection of misplaced data and protects the footer with a checksum;
+ # format 7 (requires >= 10.4) additionally supports custom compression.
+ # Changing this setting only affects new SSTs; lowering it does not convert
+ # existing files. set by jni: tableOptions.setFormatVersion
+ # https://github.com/facebook/rocksdb/blob/v10.10.1/include/rocksdb/table.h
  format_version=5
  # set by jni: tableOptions.setChecksumType
- checksum=kxxHash
+ checksum=kxxHash

bookkeeper-server/src/test/resources/conf/entry_location_rocksdb.conf

@@ -17,6 +17,13 @@
 # */
 
 [DBOptions]
+ # Keep WAL records readable by RocksDB 7.9.2 (BookKeeper 4.17.0 / Pulsar 4.0.x).
+ # Enabling track_and_verify_wals adds predecessor records (since 9.11) that
+ # 7.9.2 cannot replay. Enable only after the downgrade window has closed to
+ # detect missing/truncated WALs during recovery. This option is unknown to
+ # 7.9.2: use that release's configuration files when downgrading.
+ # https://github.com/facebook/rocksdb/blob/v10.10.1/include/rocksdb/options.h
+ track_and_verify_wals=false
  # set by jni: options.setCreateIfMissing
  create_if_missing=true
  # set by jni: options.setInfoLogLevel
@@ -57,13 +64,30 @@
 [TableOptions/BlockBasedTable "default"]
  # set by jni: tableOptions.setBlockSize
  block_size=65536
-# set by jni: tableOptions.setBlockCache, default value is: maxDirectMemory() / ledgerDirsSize / 10;
+ # An explicit LRU cache preserves the current memory/performance policy.
+ # Consider a HyperClockCache to reduce contention on concurrent reads; cache
+ # choice does not change the on-disk format. See:
+ # https://github.com/facebook/rocksdb/wiki/Block-Cache
+ # set by jni: tableOptions.setBlockCache, default value is: maxDirectMemory() / ledgerDirsSize / 10;
  block_cache=206150041
- # set by jni: tableOptions.setFormatVersion
+ # Keep newly flushed/compacted SSTs readable by RocksDB 7.9.2.
+ # Format 5 supports modern Bloom filters (readable since RocksDB 6.6).
+ # After the downgrade window closes, format 6 (requires >= 8.6) improves
+ # detection of misplaced data and protects the footer with a checksum;
+ # format 7 (requires >= 10.4) additionally supports custom compression.
+ # Changing this setting only affects new SSTs; lowering it does not convert
+ # existing files. set by jni: tableOptions.setFormatVersion
+ # https://github.com/facebook/rocksdb/blob/v10.10.1/include/rocksdb/table.h
  format_version=5
  # set by jni: tableOptions.setChecksumType
  checksum=kxxHash
+ # Ribbon filters can reduce filter memory at higher construction CPU cost
+ # and are readable by 7.9.2. Partitioned filters can reduce metadata cache
+ # pressure; decouple_partitioned_filters=true (the new default since 10.6)
+ # is also readable by 7.9.2 and need not be disabled for downgrade safety.
+ # https://github.com/facebook/rocksdb/wiki/RocksDB-Bloom-Filter
+ # https://github.com/facebook/rocksdb/wiki/Partitioned-Index-Filters
  # set by jni: tableOptions.setFilterPolicy, bloomfilter:[bits_per_key]:[use_block_based_builder]
  filter_policy=rocksdb.BloomFilter:10:false
  # set by jni: tableOptions.setCacheIndexAndFilterBlocks
- cache_index_and_filter_blocks=true
+ cache_index_and_filter_blocks=true

bookkeeper-server/src/test/resources/conf/ledger_metadata_rocksdb.conf

@@ -17,6 +17,13 @@
 # */
 
 [DBOptions]
+ # Keep WAL records readable by RocksDB 7.9.2 (BookKeeper 4.17.0 / Pulsar 4.0.x).
+ # Enabling track_and_verify_wals adds predecessor records (since 9.11) that
+ # 7.9.2 cannot replay. Enable only after the downgrade window has closed to
+ # detect missing/truncated WALs during recovery. This option is unknown to
+ # 7.9.2: use that release's configuration files when downgrading.
+ # https://github.com/facebook/rocksdb/blob/v10.10.1/include/rocksdb/options.h
+ track_and_verify_wals=false
  # set by jni: options.setCreateIfMissing
  create_if_missing=true
  # set by jni: options.setInfoLogLevel
@@ -30,7 +37,14 @@
  #no default setting in CFOptions
 
 [TableOptions/BlockBasedTable "default"]
- # set by jni: tableOptions.setFormatVersion
+ # Keep newly flushed/compacted SSTs readable by RocksDB 7.9.2.
+ # Format 5 supports modern Bloom filters (readable since RocksDB 6.6).
+ # After the downgrade window closes, format 6 (requires >= 8.6) improves
+ # detection of misplaced data and protects the footer with a checksum;
+ # format 7 (requires >= 10.4) additionally supports custom compression.
+ # Changing this setting only affects new SSTs; lowering it does not convert
+ # existing files. set by jni: tableOptions.setFormatVersion
+ # https://github.com/facebook/rocksdb/blob/v10.10.1/include/rocksdb/table.h
  format_version=5
  # set by jni: tableOptions.setChecksumType
- checksum=kxxHash
+ checksum=kxxHash

conf/default_rocksdb.conf.default

@@ -21,6 +21,13 @@
 # test case to ensure unit test coverage.
 
 [DBOptions]
+ # Keep WAL records readable by RocksDB 7.9.2 (BookKeeper 4.17.0 / Pulsar 4.0.x).
+ # Enabling track_and_verify_wals adds predecessor records (since 9.11) that
+ # 7.9.2 cannot replay. Enable only after the downgrade window has closed to
+ # detect missing/truncated WALs during recovery. This option is unknown to
+ # 7.9.2: use that release's configuration files when downgrading.
+ # https://github.com/facebook/rocksdb/blob/v10.10.1/include/rocksdb/options.h
+ track_and_verify_wals=false
  # set by jni: options.setCreateIfMissing
  create_if_missing=true
  # set by jni: options.setInfoLogLevel
@@ -34,7 +41,14 @@
  #no default setting in CFOptions
 
 [TableOptions/BlockBasedTable "default"]
- # set by jni: tableOptions.setFormatVersion
+ # Keep newly flushed/compacted SSTs readable by RocksDB 7.9.2.
+ # Format 5 supports modern Bloom filters (readable since RocksDB 6.6).
+ # After the downgrade window closes, format 6 (requires >= 8.6) improves
+ # detection of misplaced data and protects the footer with a checksum;
+ # format 7 (requires >= 10.4) additionally supports custom compression.
+ # Changing this setting only affects new SSTs; lowering it does not convert
+ # existing files. set by jni: tableOptions.setFormatVersion
+ # https://github.com/facebook/rocksdb/blob/v10.10.1/include/rocksdb/table.h
  format_version=5
  # set by jni: tableOptions.setChecksumType
- checksum=kxxHash
+ checksum=kxxHash

conf/entry_location_rocksdb.conf.default

@@ -21,6 +21,13 @@
 # test case to ensure unit test coverage.
 
 [DBOptions]
+ # Keep WAL records readable by RocksDB 7.9.2 (BookKeeper 4.17.0 / Pulsar 4.0.x).
+ # Enabling track_and_verify_wals adds predecessor records (since 9.11) that
+ # 7.9.2 cannot replay. Enable only after the downgrade window has closed to
+ # detect missing/truncated WALs during recovery. This option is unknown to
+ # 7.9.2: use that release's configuration files when downgrading.
+ # https://github.com/facebook/rocksdb/blob/v10.10.1/include/rocksdb/options.h
+ track_and_verify_wals=false
  # set by jni: options.setCreateIfMissing
  create_if_missing=true
  # set by jni: options.setInfoLogLevel
@@ -61,13 +68,30 @@
 [TableOptions/BlockBasedTable "default"]
  # set by jni: tableOptions.setBlockSize
  block_size=65536
+ # An explicit LRU cache preserves the current memory/performance policy.
+ # Consider a HyperClockCache to reduce contention on concurrent reads; cache
+ # choice does not change the on-disk format. See:
+ # https://github.com/facebook/rocksdb/wiki/Block-Cache
  # set by jni: tableOptions.setBlockCache, default value is: maxDirectMemory() / ledgerDirsSize / 10;
  block_cache=206150041
- # set by jni: tableOptions.setFormatVersion
+ # Keep newly flushed/compacted SSTs readable by RocksDB 7.9.2.
+ # Format 5 supports modern Bloom filters (readable since RocksDB 6.6).
+ # After the downgrade window closes, format 6 (requires >= 8.6) improves
+ # detection of misplaced data and protects the footer with a checksum;
+ # format 7 (requires >= 10.4) additionally supports custom compression.
+ # Changing this setting only affects new SSTs; lowering it does not convert
+ # existing files. set by jni: tableOptions.setFormatVersion
+ # https://github.com/facebook/rocksdb/blob/v10.10.1/include/rocksdb/table.h
  format_version=5
  # set by jni: tableOptions.setChecksumType
  checksum=kxxHash
+ # Ribbon filters can reduce filter memory at higher construction CPU cost
+ # and are readable by 7.9.2. Partitioned filters can reduce metadata cache
+ # pressure; decouple_partitioned_filters=true (the new default since 10.6)
+ # is also readable by 7.9.2 and need not be disabled for downgrade safety.
+ # https://github.com/facebook/rocksdb/wiki/RocksDB-Bloom-Filter
+ # https://github.com/facebook/rocksdb/wiki/Partitioned-Index-Filters
  # set by jni: tableOptions.setFilterPolicy, bloomfilter:[bits_per_key]:[use_block_based_builder]
  filter_policy=rocksdb.BloomFilter:10:false
  # set by jni: tableOptions.setCacheIndexAndFilterBlocks
- cache_index_and_filter_blocks=true
+ cache_index_and_filter_blocks=true

conf/ledger_metadata_rocksdb.conf.default

@@ -21,6 +21,13 @@
 # test case to ensure unit test coverage.
 
 [DBOptions]
+ # Keep WAL records readable by RocksDB 7.9.2 (BookKeeper 4.17.0 / Pulsar 4.0.x).
+ # Enabling track_and_verify_wals adds predecessor records (since 9.11) that
+ # 7.9.2 cannot replay. Enable only after the downgrade window has closed to
+ # detect missing/truncated WALs during recovery. This option is unknown to
+ # 7.9.2: use that release's configuration files when downgrading.
+ # https://github.com/facebook/rocksdb/blob/v10.10.1/include/rocksdb/options.h
+ track_and_verify_wals=false
  # set by jni: options.setCreateIfMissing
  create_if_missing=true
  # set by jni: options.setInfoLogLevel
@@ -34,7 +41,14 @@
  #no default setting in CFOptions
 
 [TableOptions/BlockBasedTable "default"]
- # set by jni: tableOptions.setFormatVersion
+ # Keep newly flushed/compacted SSTs readable by RocksDB 7.9.2.
+ # Format 5 supports modern Bloom filters (readable since RocksDB 6.6).
+ # After the downgrade window closes, format 6 (requires >= 8.6) improves
+ # detection of misplaced data and protects the footer with a checksum;
+ # format 7 (requires >= 10.4) additionally supports custom compression.
+ # Changing this setting only affects new SSTs; lowering it does not convert
+ # existing files. set by jni: tableOptions.setFormatVersion
+ # https://github.com/facebook/rocksdb/blob/v10.10.1/include/rocksdb/table.h
  format_version=5
  # set by jni: tableOptions.setChecksumType
- checksum=kxxHash
+ checksum=kxxHash

pom.xml

@@ -188,7 +188,7 @@
     <protoc.version>${protobuf.version}</protoc.version>
     <protoc-gen-grpc-java.version>${grpc.version}</protoc-gen-grpc-java.version>
     <reflections.version>0.9.11</reflections.version>
-    <rocksdb.version>9.9.3</rocksdb.version>
+    <rocksdb.version>10.10.1.1</rocksdb.version>
     <shrinkwrap.version>3.3.0</shrinkwrap.version>
     <slf4j.version>2.0.12</slf4j.version>
     <slog.version>0.9.9</slog.version>