diff --git a/AGENTS.md b/AGENTS.md index 682c0170515..1c6c0b020d6 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -135,6 +135,8 @@ When creating an **issue** or **pull request**, read the repo templates first an - **Issue:** `.github/ISSUE_TEMPLATE/` β€” pick the template that matches the request (bug, enhancement, feature, question, performance). - **PR:** `.github/pull_request_template.md` β€” use it for the PR title/body, checklists, and release note. +When creating git commits, always use `-s` / `--signoff` so the commit message includes a `Signed-off-by` trailer (DCO). + Typical fork workflow: branch from `upstream/master`, push to `origin`, open the PR against `pingcap/tiflash` (`master`). ## πŸ“– References diff --git a/contrib/cloud-storage-engine b/contrib/cloud-storage-engine index 9cbffb66ae7..b2572e84bbb 160000 --- a/contrib/cloud-storage-engine +++ b/contrib/cloud-storage-engine @@ -1 +1 @@ -Subproject commit 9cbffb66ae79b72212cef4622b60d0c5e766856b +Subproject commit b2572e84bbb727e691dc8624b4f7edb888fc91e0 diff --git a/contrib/tiflash-columnar-hub/Cargo.lock b/contrib/tiflash-columnar-hub/Cargo.lock index aff3db20b03..b451a4340a2 100644 --- a/contrib/tiflash-columnar-hub/Cargo.lock +++ b/contrib/tiflash-columnar-hub/Cargo.lock @@ -3770,7 +3770,7 @@ dependencies = [ [[package]] name = "kvproto" version = "0.0.2" -source = "git+https://github.com/pingcap/kvproto.git?rev=683dad8fa3689deb243f2ff8ab5847c97af53e38#683dad8fa3689deb243f2ff8ab5847c97af53e38" +source = "git+https://github.com/pingcap/kvproto.git?rev=9327469#9327469bb3ce7bec31507d3090b39d2127f019d3" dependencies = [ "bytes", "futures 0.3.32", @@ -7674,7 +7674,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "97fee6b57c6a41524a810daee9286c02d7752c4253064d0b05472833a438f675" dependencies = [ "cfg-if", - "rand 0.7.3", + "rand 0.8.6", "static_assertions", ] diff --git a/contrib/tiflash-columnar-hub/Cargo.toml b/contrib/tiflash-columnar-hub/Cargo.toml index d12f4a12e56..a61d4fbde71 100644 --- a/contrib/tiflash-columnar-hub/Cargo.toml +++ b/contrib/tiflash-columnar-hub/Cargo.toml @@ -10,7 +10,7 @@ cloud_encryption = { git = "https://github.com/tidbcloud/cloud-storage-engine.gi keys = { git = "https://github.com/tidbcloud/cloud-storage-engine.git", branch = "cloud-engine" } kvengine = { git = "https://github.com/tidbcloud/cloud-storage-engine.git", branch = "cloud-engine" } kvenginepb = { git = "https://github.com/tidbcloud/cloud-storage-engine.git", branch = "cloud-engine" } -kvproto = { git = "https://github.com/pingcap/kvproto.git", rev = "683dad8fa3689deb243f2ff8ab5847c97af53e38" } +kvproto = { git = "https://github.com/pingcap/kvproto.git", rev = "9327469" } pd_client = { git = "https://github.com/tidbcloud/cloud-storage-engine.git", branch = "cloud-engine", default-features = false } prometheus = { version = "=0.13.0", features = ["nightly", "push"], default-features = true } security = { git = "https://github.com/tidbcloud/cloud-storage-engine.git", branch = "cloud-engine", default-features = false } diff --git a/contrib/tiflash-columnar-hub/hub-runtime/ffi/src/RaftStoreProxyFFI/ProxyFFI.h b/contrib/tiflash-columnar-hub/hub-runtime/ffi/src/RaftStoreProxyFFI/ProxyFFI.h index 4eff355463a..54f59b08a3f 100644 --- a/contrib/tiflash-columnar-hub/hub-runtime/ffi/src/RaftStoreProxyFFI/ProxyFFI.h +++ b/contrib/tiflash-columnar-hub/hub-runtime/ffi/src/RaftStoreProxyFFI/ProxyFFI.h @@ -251,7 +251,7 @@ struct CloudStorageEngineInterfaces { ColumnarReaderPtr (*fn_get_columnar_reader)(uint64_t, uint64_t, uint64_t, BaseBuffView, BaseBuffView, BaseBuffView, BaseBuffView, - BaseBuffView, BaseBuffView, + BaseBuffView, BaseBuffView, bool, RaftStoreProxyPtr); uint64_t (*fn_read_block)(ColumnarReaderPtr, uint64_t); RustStrWithView (*fn_read_handle)(ColumnarReaderPtr); diff --git a/contrib/tiflash-columnar-hub/hub-runtime/src/cloud_helper.rs b/contrib/tiflash-columnar-hub/hub-runtime/src/cloud_helper.rs index 3310cc50edd..7f87a529199 100644 --- a/contrib/tiflash-columnar-hub/hub-runtime/src/cloud_helper.rs +++ b/contrib/tiflash-columnar-hub/hub-runtime/src/cloud_helper.rs @@ -527,6 +527,7 @@ impl CloudHelper { filter_conditions: Vec, ann_query_info: tipb::AnnQueryInfo, fts_query_info: tipb::FtsQueryInfo, + enable_trim_minmax: bool, ) -> Result { let dfs = self.dfs.clone(); let pd_client = self.pd_client.clone(); @@ -537,12 +538,14 @@ impl CloudHelper { let ia_ctx = self.ia_ctx.clone(); let http_client = self.http_client.clone(); info!( - "make_columnar_reader, start_ts: {:?}, filter_conditions: {:?}, num_tables: {:?}", + "make_columnar_reader, start_ts: {:?}, filter_conditions: {:?}, num_tables: {:?}, enable_trim_minmax: {:?}", start_ts, filter_conditions, - tables.len() + tables.len(), + enable_trim_minmax ); - let scan_ctx = TableScanCtx::new(table_scan, filter_conditions); + let scan_ctx = + TableScanCtx::new(table_scan, filter_conditions).with_enable_trim_minmax(enable_trim_minmax); let (tx, rx) = tikv_util::mpsc::bounded(1); let start = Instant::now_coarse(); let vector_index_cache = self.vector_index_cache.clone(); diff --git a/contrib/tiflash-columnar-hub/hub-runtime/src/columnar_impls.rs b/contrib/tiflash-columnar-hub/hub-runtime/src/columnar_impls.rs index 20661db32ce..cd29c8ff829 100644 --- a/contrib/tiflash-columnar-hub/hub-runtime/src/columnar_impls.rs +++ b/contrib/tiflash-columnar-hub/hub-runtime/src/columnar_impls.rs @@ -128,6 +128,7 @@ pub unsafe extern "C" fn ffi_make_columnar_reader( filter_conditions: BaseBuffView, ann_query_info: BaseBuffView, fts_query_info: BaseBuffView, + enable_trim_minmax: bool, hub_ptr: RaftStoreProxyPtr, ) -> ColumnarReaderPtr { let mut cols_pb = tipb::TableInfo::default(); @@ -218,6 +219,7 @@ pub unsafe extern "C" fn ffi_make_columnar_reader( filter_conditions_pb, ann_query_info_pb, fts_query_info_pb, + enable_trim_minmax, ) { Ok(reader) => (Box::into_raw(Box::new(reader)) as RawVoidPtr).into(), Err(err) => err.into(), diff --git a/contrib/tiflash-columnar-hub/hub-runtime/src/interfaces.rs b/contrib/tiflash-columnar-hub/hub-runtime/src/interfaces.rs index 94d4f123292..8c9ca9d3af9 100644 --- a/contrib/tiflash-columnar-hub/hub-runtime/src/interfaces.rs +++ b/contrib/tiflash-columnar-hub/hub-runtime/src/interfaces.rs @@ -393,7 +393,8 @@ pub mod root { arg7: root::DB::BaseBuffView, arg8: root::DB::BaseBuffView, arg9: root::DB::BaseBuffView, - arg10: root::DB::RaftStoreProxyPtr, + arg10: bool, + arg11: root::DB::RaftStoreProxyPtr, ) -> root::DB::ColumnarReaderPtr, >, pub fn_read_block: ::std::option::Option< diff --git a/dbms/src/Common/TiFlashMetrics.h b/dbms/src/Common/TiFlashMetrics.h index 1d709eb7f30..a9a64cf5244 100644 --- a/dbms/src/Common/TiFlashMetrics.h +++ b/dbms/src/Common/TiFlashMetrics.h @@ -429,6 +429,38 @@ static_assert(RAFT_REGION_BIG_WRITE_THRES * 4 < RAFT_REGION_BIG_WRITE_MAX, "Inva "Bucketed histogram of rough set filter rate", \ Histogram, \ F(type_dtfile_pack, {{"type", "dtfile_pack"}}, EqualWidthBuckets{0, 6, 20})) \ + M(tiflash_storage_rough_set_pack_count, \ + "Total number of packs before and after query rough set filtering", \ + Counter, \ + F(stage_query_input, {"stage", "query_input"}), \ + F(stage_query_filtered, {"stage", "query_filtered"}), \ + F(stage_query_remaining, {"stage", "query_remaining"})) \ + M(tiflash_storage_trim_minmax_select_count, \ + "Total number of trim min-max selection results", \ + Counter, \ + F(result_used, {"result", "used"}), \ + F(result_fallback_disabled, {"result", "fallback_disabled"}), \ + F(result_fallback_non_meta_v2, {"result", "fallback_non_meta_v2"}), \ + F(result_fallback_column_missing, {"result", "fallback_column_missing"}), \ + F(result_fallback_no_meta, {"result", "fallback_no_meta"}), \ + F(result_fallback_unsupported_version, {"result", "fallback_unsupported_version"}), \ + F(result_fallback_metadata_mismatch, {"result", "fallback_metadata_mismatch"}), \ + F(result_fallback_index_missing, {"result", "fallback_index_missing"}), \ + F(result_fallback_unsupported_expression, {"result", "fallback_unsupported_expression"}), \ + F(result_fallback_predicate_outside_range, {"result", "fallback_predicate_outside_range"}), \ + F(result_fallback_invalid_pack_marks, {"result", "fallback_invalid_pack_marks"})) \ + M(tiflash_storage_trim_minmax_rough_check_pack_count, \ + "Total number of packs by trim min-max rough check result", \ + Counter, \ + F(result_none, {"result", "none"}), \ + F(result_some, {"result", "some"}), \ + F(result_all, {"result", "all"}), \ + F(result_all_null, {"result", "all_null"})) \ + M(tiflash_storage_trim_minmax_correction_pack_count, \ + "Total number of packs whose trim min-max rough check result is conservatively corrected", \ + Counter, \ + F(type_none_to_some, {"type", "none_to_some"}), \ + F(type_all_to_some, {"type", "all_to_some"})) \ M(tiflash_disaggregated_object_lock_request_count, \ "Total number of S3 object lock/delete request", \ Counter, \ diff --git a/dbms/src/Interpreters/Settings.h b/dbms/src/Interpreters/Settings.h index 4388a78d5d5..4b97a92844b 100644 --- a/dbms/src/Interpreters/Settings.h +++ b/dbms/src/Interpreters/Settings.h @@ -177,6 +177,7 @@ struct Settings M(SettingFloat, dt_bg_gc_delta_delete_ratio_to_trigger_gc, 0.3, "Trigger segment's gc when the ratio of delta delete range to stable exceeds this ratio.") \ M(SettingBool, dt_enable_logical_split, false, "Enable logical split or not in DeltaTree Engine.") \ M(SettingBool, dt_enable_rough_set_filter, true, "Whether to parse where expression as Rough Set Index filter or not.") \ + M(SettingBool, dt_enable_trim_minmax, false, "Whether to generate and use trim min-max index for DATE/DATETIME/TIMESTAMP Rough Set filtering.") \ M(SettingBool, dt_enable_relevant_place, false, "Enable relevant place or not in DeltaTree Engine.") \ M(SettingBool, dt_enable_skippable_place, true, "Enable skippable place or not in DeltaTree Engine.") \ M(SettingBool, dt_enable_stable_column_cache, true, "Enable column cache for StorageDeltaMerge.") \ diff --git a/dbms/src/Storages/DeltaMerge/File/ColumnStat.h b/dbms/src/Storages/DeltaMerge/File/ColumnStat.h index 8156b29f674..d359c1750dc 100644 --- a/dbms/src/Storages/DeltaMerge/File/ColumnStat.h +++ b/dbms/src/Storages/DeltaMerge/File/ColumnStat.h @@ -22,6 +22,8 @@ #include #include +#include + namespace DB::DM { struct ColumnStat @@ -44,6 +46,9 @@ struct ColumnStat std::vector indexes{}; + // Optional trim min-max metadata. Independent from `indexes` / local-index lifecycle. + std::optional trim_minmax_index{}; + #ifndef NDEBUG // This field is only used for testing String additional_data_for_test{}; @@ -71,6 +76,9 @@ struct ColumnStat pb_idx->CopyFrom(idx); } + if (trim_minmax_index.has_value()) + *stat.mutable_trim_minmax_index() = *trim_minmax_index; + #ifndef NDEBUG stat.set_additional_data_for_test(additional_data_for_test); #endif @@ -131,6 +139,13 @@ struct ColumnStat indexes.emplace_back(pb_idx); } + // Soft-load only. Structural validation and fallback happen at selection time so a + // corrupt / unsupported trim meta never fails DMFile open. + if (proto.has_trim_minmax_index()) + trim_minmax_index = proto.trim_minmax_index(); + else + trim_minmax_index.reset(); + #ifndef NDEBUG additional_data_for_test = proto.additional_data_for_test(); #endif @@ -237,6 +252,7 @@ readText(ColumnStats & column_sats, DMFileFormat::Version ver, ReadBuffer & buf) .serialized_bytes = serialized_bytes, // ... here ignore some fields with default initializers .indexes = {}, + .trim_minmax_index = {}, #ifndef NDEBUG .additional_data_for_test = {}, #endif diff --git a/dbms/src/Storages/DeltaMerge/File/DMFile.cpp b/dbms/src/Storages/DeltaMerge/File/DMFile.cpp index b9750c77983..678f03e1556 100644 --- a/dbms/src/Storages/DeltaMerge/File/DMFile.cpp +++ b/dbms/src/Storages/DeltaMerge/File/DMFile.cpp @@ -192,6 +192,12 @@ String DMFile::colIndexCacheKey(const FileNameBase & file_name_base) const return colIndexPath(file_name_base); } +String DMFile::colTrimIndexCacheKey(const FileNameBase & file_name_base) const +{ + // Distinct from ordinary `.idx` cache key; path already ends with `.trim.idx`. + return colTrimIndexPath(file_name_base); +} + String DMFile::colMarkCacheKey(const FileNameBase & file_name_base) const { return colMarkPath(file_name_base); @@ -288,6 +294,11 @@ EncryptionPath DMFile::encryptionIndexPath(const FileNameBase & file_name_base) return EncryptionPath(encryptionBasePath(), file_name_base + details::INDEX_FILE_SUFFIX, keyspaceId()); } +EncryptionPath DMFile::encryptionTrimIndexPath(const FileNameBase & file_name_base) const +{ + return EncryptionPath(encryptionBasePath(), file_name_base + details::TRIM_INDEX_FILE_SUFFIX, keyspaceId()); +} + EncryptionPath DMFile::encryptionMarkPath(const FileNameBase & file_name_base) const { return EncryptionPath(encryptionBasePath(), file_name_base + details::MARK_FILE_SUFFIX, keyspaceId()); diff --git a/dbms/src/Storages/DeltaMerge/File/DMFile.h b/dbms/src/Storages/DeltaMerge/File/DMFile.h index c97dc7295eb..f84ece41ac4 100644 --- a/dbms/src/Storages/DeltaMerge/File/DMFile.h +++ b/dbms/src/Storages/DeltaMerge/File/DMFile.h @@ -290,17 +290,23 @@ class DMFile : private boost::noncopyable { return subFilePath(colIndexFileName(file_name_base)); } + String colTrimIndexPath(const FileNameBase & file_name_base) const + { + return subFilePath(colTrimIndexFileName(file_name_base)); + } String colMarkPath(const FileNameBase & file_name_base) const { return subFilePath(colMarkFileName(file_name_base)); } String colIndexCacheKey(const FileNameBase & file_name_base) const; + String colTrimIndexCacheKey(const FileNameBase & file_name_base) const; String colMarkCacheKey(const FileNameBase & file_name_base) const; String encryptionBasePath() const; EncryptionPath encryptionDataPath(const FileNameBase & file_name_base) const; EncryptionPath encryptionIndexPath(const FileNameBase & file_name_base) const; + EncryptionPath encryptionTrimIndexPath(const FileNameBase & file_name_base) const; EncryptionPath encryptionMarkPath(const FileNameBase & file_name_base) const; static FileNameBase getFileNameBase(ColId col_id, const IDataType::SubstreamPath & substream = {}) diff --git a/dbms/src/Storages/DeltaMerge/File/DMFileBlockInputStream.cpp b/dbms/src/Storages/DeltaMerge/File/DMFileBlockInputStream.cpp index 93efb0fe9a3..579c8b40c6f 100644 --- a/dbms/src/Storages/DeltaMerge/File/DMFileBlockInputStream.cpp +++ b/dbms/src/Storages/DeltaMerge/File/DMFileBlockInputStream.cpp @@ -88,7 +88,9 @@ DMFileBlockInputStreamPtr DMFileBlockInputStreamBuilder::buildNoLocalIndex( rowkey_ranges, EMPTY_RS_OPERATOR, read_packs, - tracing_id); + tracing_id, + enable_trim_minmax, + read_tag); } DMFileReader reader( @@ -137,6 +139,7 @@ SkippableBlockInputStreamPtr createSimpleBlockInputStream( DMFileBlockInputStreamBuilder & DMFileBlockInputStreamBuilder::setFromSettings(const Settings & settings) { enable_column_cache = settings.dt_enable_stable_column_cache; + enable_trim_minmax = settings.dt_enable_trim_minmax; max_read_buffer_size = settings.max_read_buffer_size; max_sharing_column_bytes_for_all = settings.dt_max_sharing_column_bytes_for_all; return *this; @@ -210,7 +213,9 @@ SkippableBlockInputStreamPtr DMFileBlockInputStreamBuilder::buildForVectorIndex( rowkey_ranges, EMPTY_RS_OPERATOR, read_packs, - tracing_id); + tracing_id, + enable_trim_minmax, + read_tag); } DMFileReader rest_columns_reader( @@ -296,7 +301,9 @@ SkippableBlockInputStreamPtr DMFileBlockInputStreamBuilder::buildForFullTextInde rowkey_ranges, EMPTY_RS_OPERATOR, read_packs, - tracing_id); + tracing_id, + enable_trim_minmax, + read_tag); } DMFileReader rest_columns_reader( diff --git a/dbms/src/Storages/DeltaMerge/File/DMFileBlockInputStream.h b/dbms/src/Storages/DeltaMerge/File/DMFileBlockInputStream.h index 5ec5f660765..eafabad6910 100644 --- a/dbms/src/Storages/DeltaMerge/File/DMFileBlockInputStream.h +++ b/dbms/src/Storages/DeltaMerge/File/DMFileBlockInputStream.h @@ -235,10 +235,10 @@ class DMFileBlockInputStreamBuilder FileProviderPtr file_provider; // clean read - bool enable_handle_clean_read = false; bool is_fast_scan = false; bool enable_del_clean_read = false; + bool enable_trim_minmax = false; UInt64 max_data_version = std::numeric_limits::max(); // packs filter (filter by pack index) IdSetPtr read_packs; diff --git a/dbms/src/Storages/DeltaMerge/File/DMFileBlockOutputStream.cpp b/dbms/src/Storages/DeltaMerge/File/DMFileBlockOutputStream.cpp index 5c6872a1b6c..b77d373426a 100644 --- a/dbms/src/Storages/DeltaMerge/File/DMFileBlockOutputStream.cpp +++ b/dbms/src/Storages/DeltaMerge/File/DMFileBlockOutputStream.cpp @@ -31,7 +31,9 @@ DMFileBlockOutputStream::DMFileBlockOutputStream( context.getSettingsRef().dt_compression_method, context.getSettingsRef().dt_compression_level), context.getSettingsRef().min_compress_block_size, - context.getSettingsRef().max_compress_block_size}) + context.getSettingsRef().max_compress_block_size, + context.getSettingsRef().dt_enable_trim_minmax, + }) {} -} // namespace DB::DM \ No newline at end of file +} // namespace DB::DM diff --git a/dbms/src/Storages/DeltaMerge/File/DMFileMeta.cpp b/dbms/src/Storages/DeltaMerge/File/DMFileMeta.cpp index 4a4169cbce1..ef159321ce8 100644 --- a/dbms/src/Storages/DeltaMerge/File/DMFileMeta.cpp +++ b/dbms/src/Storages/DeltaMerge/File/DMFileMeta.cpp @@ -25,6 +25,7 @@ namespace DB::ErrorCodes { extern const int BAD_ARGUMENTS; +extern const int LOGICAL_ERROR; } // namespace DB::ErrorCodes namespace DB::DM @@ -53,6 +54,10 @@ void DMFileMeta::initializeIndices() directory.list(sub_files); for (const auto & name : sub_files) { + // Skip trim min-max files. Their names also end with `.idx` (`*.trim.idx`), + // but the prefix is not a plain ColId and must not enter column_indices. + if (endsWith(name, details::TRIM_INDEX_FILE_SUFFIX)) + continue; if (endsWith(name, details::INDEX_FILE_SUFFIX)) { column_indices.insert( @@ -405,6 +410,14 @@ UInt64 DMFileMeta::getFileSize(ColId col_id, const String & filename) const { auto itr = column_stats.find(col_id); RUNTIME_CHECK(itr != column_stats.end(), col_id); + // Trim index size is only available from MergedSubFileInfo, never from ColumnStat. + if (endsWith(filename, details::TRIM_INDEX_FILE_SUFFIX)) + { + throw Exception( + ErrorCodes::LOGICAL_ERROR, + "trim index size must be read from MergedSubFileInfo, filename={}", + filename); + } if (endsWith(filename, ".idx")) { return itr->second.index_bytes; diff --git a/dbms/src/Storages/DeltaMerge/File/DMFileMetaV2.cpp b/dbms/src/Storages/DeltaMerge/File/DMFileMetaV2.cpp index 700c6bb214a..1094bea5007 100644 --- a/dbms/src/Storages/DeltaMerge/File/DMFileMetaV2.cpp +++ b/dbms/src/Storages/DeltaMerge/File/DMFileMetaV2.cpp @@ -109,7 +109,7 @@ void DMFileMetaV2::parseColumnStat(std::string_view buffer) ColumnStat stat; stat.parseFromBuffer(rbuf); // Do not overwrite the ColumnStat if already exist, it may - // created by `ExteandColumnStat` + // created by `ExtendColumnStat` column_stats.emplace(stat.col_id, std::move(stat)); } } @@ -181,7 +181,7 @@ void DMFileMetaV2::finalize( writeSLPackStatToBuffer(tmp_buffer), writeSLPackPropertyToBuffer(tmp_buffer), writeExtendColumnStatToBuffer(tmp_buffer), - writeMergedSubFilePosotionsToBuffer(tmp_buffer), + writeMergedSubFilePositionsToBuffer(tmp_buffer), }; writePODBinary(meta_block_handles, tmp_buffer); writeIntBinary(static_cast(meta_block_handles.size()), tmp_buffer); @@ -255,7 +255,7 @@ DMFileMeta::BlockHandle DMFileMetaV2::writeExtendColumnStatToBuffer(WriteBuffer return BlockHandle{BlockType::ExtendColumnStat, offset, buffer.count() - offset}; } -DMFileMeta::BlockHandle DMFileMetaV2::writeMergedSubFilePosotionsToBuffer(WriteBuffer & buffer) +DMFileMeta::BlockHandle DMFileMetaV2::writeMergedSubFilePositionsToBuffer(WriteBuffer & buffer) { auto offset = buffer.count(); diff --git a/dbms/src/Storages/DeltaMerge/File/DMFileMetaV2.h b/dbms/src/Storages/DeltaMerge/File/DMFileMetaV2.h index d3612c945d0..31ef430fcc8 100644 --- a/dbms/src/Storages/DeltaMerge/File/DMFileMetaV2.h +++ b/dbms/src/Storages/DeltaMerge/File/DMFileMetaV2.h @@ -125,7 +125,7 @@ class DMFileMetaV2 : public DMFileMeta BlockHandle writeSLPackPropertyToBuffer(WriteBuffer & buffer) const; BlockHandle writeColumnStatToBuffer(WriteBuffer & buffer); BlockHandle writeExtendColumnStatToBuffer(WriteBuffer & buffer); - BlockHandle writeMergedSubFilePosotionsToBuffer(WriteBuffer & buffer); + BlockHandle writeMergedSubFilePositionsToBuffer(WriteBuffer & buffer); // read void parse(std::string_view buffer); diff --git a/dbms/src/Storages/DeltaMerge/File/DMFilePackFilter.cpp b/dbms/src/Storages/DeltaMerge/File/DMFilePackFilter.cpp index 61408939a3f..292528c7c17 100644 --- a/dbms/src/Storages/DeltaMerge/File/DMFilePackFilter.cpp +++ b/dbms/src/Storages/DeltaMerge/File/DMFilePackFilter.cpp @@ -17,16 +17,62 @@ #include #include #include +#include #include +#include +#include #include #include #include +#include #include #include #include namespace DB::DM { +namespace +{ +void recordTrimMinMaxSelect(TrimMinMaxFallbackReason reason) +{ + switch (reason) + { + case TrimMinMaxFallbackReason::None: + GET_METRIC(tiflash_storage_trim_minmax_select_count, result_used).Increment(); + return; + case TrimMinMaxFallbackReason::Disabled: + GET_METRIC(tiflash_storage_trim_minmax_select_count, result_fallback_disabled).Increment(); + return; + case TrimMinMaxFallbackReason::NonMetaV2: + GET_METRIC(tiflash_storage_trim_minmax_select_count, result_fallback_non_meta_v2).Increment(); + return; + case TrimMinMaxFallbackReason::ColumnMissing: + GET_METRIC(tiflash_storage_trim_minmax_select_count, result_fallback_column_missing).Increment(); + return; + case TrimMinMaxFallbackReason::NoMeta: + GET_METRIC(tiflash_storage_trim_minmax_select_count, result_fallback_no_meta).Increment(); + return; + case TrimMinMaxFallbackReason::UnsupportedVersion: + GET_METRIC(tiflash_storage_trim_minmax_select_count, result_fallback_unsupported_version).Increment(); + return; + case TrimMinMaxFallbackReason::MetadataMismatch: + GET_METRIC(tiflash_storage_trim_minmax_select_count, result_fallback_metadata_mismatch).Increment(); + return; + case TrimMinMaxFallbackReason::IndexMissing: + GET_METRIC(tiflash_storage_trim_minmax_select_count, result_fallback_index_missing).Increment(); + return; + case TrimMinMaxFallbackReason::UnsupportedExpression: + GET_METRIC(tiflash_storage_trim_minmax_select_count, result_fallback_unsupported_expression).Increment(); + return; + case TrimMinMaxFallbackReason::PredicateBoundaryOutsideRange: + GET_METRIC(tiflash_storage_trim_minmax_select_count, result_fallback_predicate_outside_range).Increment(); + return; + case TrimMinMaxFallbackReason::InvalidPackMarks: + GET_METRIC(tiflash_storage_trim_minmax_select_count, result_fallback_invalid_pack_marks).Increment(); + return; + } +} +} // namespace DMFilePackFilter::MatchDetails DMFilePackFilter::loadValidRowsAndBytes( const DMContext & dm_context, @@ -56,6 +102,7 @@ DMFilePackFilterResultPtr DMFilePackFilter::load() SCOPE_EXIT({ scan_context->total_rs_pack_filter_check_time_ns += watch.elapsed(); }); size_t pack_count = dmfile->getPacks(); DMFilePackFilterResult result(index_cache, read_limiter, pack_count); + result.param.record_trim_metrics = read_tag == ReadTag::Query; auto read_all_packs = (rowkey_ranges.size() == 1 && rowkey_ranges[0].all()) || rowkey_ranges.empty(); if (!read_all_packs) { @@ -126,12 +173,9 @@ DMFilePackFilterResultPtr DMFilePackFilter::load() /// Check packs by filter in where clause if (filter) { - // Load index based on filter. - ColIds ids = filter->getColumnIDs(); - for (const auto & id : ids) - { - tryLoadIndex(result.param, id); - } + // Load index based on filter requests (normal and/or PreferTrim). + for (const auto & request : filter->getIndexRequests()) + tryLoadIndexByRequest(result.param, request); const auto check_results = filter->roughCheck(0, pack_count, result.param); std::transform( @@ -159,6 +203,19 @@ DMFilePackFilterResultPtr DMFilePackFilter::load() scan_context->rs_pack_filter_some += some_count; scan_context->rs_pack_filter_all += all_count; scan_context->rs_pack_filter_all_null += all_null_count; + + if (filter) + { + if (after_read_packs != 0) + GET_METRIC(tiflash_storage_rough_set_pack_count, stage_query_input).Increment(after_read_packs); + if (after_read_packs != after_filter) + { + GET_METRIC(tiflash_storage_rough_set_pack_count, stage_query_filtered) + .Increment(after_read_packs - after_filter); + } + if (after_filter != 0) + GET_METRIC(tiflash_storage_rough_set_pack_count, stage_query_remaining).Increment(after_filter); + } } Float64 filter_rate = 0.0; @@ -377,6 +434,125 @@ void DMFilePackFilter::tryLoadIndex(RSCheckParam & param, ColId col_id) loadIndex(param.indexes, dmfile, file_provider, index_cache, set_cache_if_miss, col_id, read_limiter, scan_context); } +void DMFilePackFilter::tryLoadIndexByRequest(RSCheckParam & param, const RSIndexRequest & request) +{ + if (request.preferred_kind == RSIndexKind::PreferTrim) + { + const auto reason = request.query_domain.has_value() + ? tryLoadTrimIndex(param, request.col_id, *request.query_domain) + : TrimMinMaxFallbackReason::UnsupportedExpression; + if (param.record_trim_metrics) + recordTrimMinMaxSelect(reason); + if (reason == TrimMinMaxFallbackReason::None) + return; + } + tryLoadIndex(param, request.col_id); +} + +TrimMinMaxFallbackReason DMFilePackFilter::tryLoadTrimIndex( + RSCheckParam & param, + ColId col_id, + const DateQueryDomain & query_domain) +{ + if (!enable_trim_minmax) + return TrimMinMaxFallbackReason::Disabled; + + if (!dmfile->useMetaV2()) + return TrimMinMaxFallbackReason::NonMetaV2; + + if (!dmfile->isColumnExist(col_id)) + return TrimMinMaxFallbackReason::ColumnMissing; + + const auto & col_stat = dmfile->getColumnStat(col_id); + const auto * dmfile_meta = typeid_cast(dmfile->meta.get()); + if (!dmfile_meta) + return TrimMinMaxFallbackReason::MetadataMismatch; + + const auto file_name_base = DMFile::getFileNameBase(col_id); + const auto trim_fname = colTrimIndexFileName(file_name_base); + + // If trim is already loaded for this column, still require THIS predicate's + // query domain to be trim-eligible. Otherwise fall through so the caller + // loads the ordinary min-max for the non-eligible request. + if (auto it = param.trim_indexes.find(col_id); it != param.trim_indexes.end()) + { + return query_domain.isTrimEligible(it->second.meta.lower_bound, it->second.meta.upper_bound) + ? TrimMinMaxFallbackReason::None + : TrimMinMaxFallbackReason::PredicateBoundaryOutsideRange; + } + + TrimMinMaxIndexMeta meta; + auto reason = TrimMinMax::trySelectTrimMeta( + /*read_enabled*/ true, + col_stat.trim_minmax_index, + *col_stat.type, + dmfile->getPacks(), + dmfile_meta->merged_sub_file_infos, + trim_fname, + &meta); + if (reason != TrimMinMaxFallbackReason::None) + return reason; + + if (!query_domain.isTrimEligible(meta.lower_bound, meta.upper_bound)) + return TrimMinMaxFallbackReason::PredicateBoundaryOutsideRange; + + auto load_trim = [&]() -> MinMaxIndexPtr { + const auto file_path = dmfile->meta->mergedPath(meta.merged_file_number); + const auto offset = meta.merged_file_offset; + const auto data_size = meta.file_size; + + auto index_guard = S3::S3RandomAccessFile::setReadFileInfo({ + .size = dmfile->getReadFileSize(col_id, trim_fname), + .scan_context = scan_context, + }); + + auto buffer = ReadBufferFromRandomAccessFileBuilder::build( + file_provider, + file_path, + dmfile_meta->encryptionMergedPath(meta.merged_file_number), + std::min(data_size, dmfile->getConfiguration()->getChecksumFrameLength()), + read_limiter); + auto ret = buffer.seek(offset); + RUNTIME_CHECK_MSG( + ret >= 0, + "Failed to seek in merged file for trim index, ret={} file_path={} offset={}", + ret, + file_path, + offset); + + String raw_data(data_size, '\0'); + buffer.read(reinterpret_cast(raw_data.data()), data_size); + + auto buf = ChecksumReadBufferBuilder::build( + std::move(raw_data), + file_path, + dmfile->getConfiguration()->getChecksumAlgorithm(), + dmfile->getConfiguration()->getChecksumFrameLength()); + + auto header_size = dmfile->getConfiguration()->getChecksumHeaderLength(); + auto frame_total_size = dmfile->getConfiguration()->getChecksumFrameLength() + header_size; + auto frame_count = data_size / frame_total_size + (data_size % frame_total_size != 0); + return MinMaxIndex::read(*col_stat.type, *buf, data_size - header_size * frame_count); + }; + + MinMaxIndexPtr minmax_index; + if (index_cache && set_cache_if_miss) + minmax_index = index_cache->getOrSet(dmfile->colTrimIndexCacheKey(file_name_base), load_trim); + else + { + if (index_cache) + minmax_index = index_cache->get(dmfile->colTrimIndexCacheKey(file_name_base)); + if (minmax_index == nullptr) + minmax_index = load_trim(); + } + + if (!minmax_index || !TrimMinMax::validateTrimPackMarks(minmax_index->packMarks(), meta.pack_count)) + return TrimMinMaxFallbackReason::InvalidPackMarks; + + param.trim_indexes.emplace(col_id, TrimRSIndex{.type = col_stat.type, .minmax = minmax_index, .meta = meta}); + return TrimMinMaxFallbackReason::None; +} + std::pair, DMFilePackFilterResults> DMFilePackFilter::getSkippedRangeAndFilter( const DMContext & dm_context, const DMFiles & dmfiles, diff --git a/dbms/src/Storages/DeltaMerge/File/DMFilePackFilter.h b/dbms/src/Storages/DeltaMerge/File/DMFilePackFilter.h index 738660f1f32..32c73c299ab 100644 --- a/dbms/src/Storages/DeltaMerge/File/DMFilePackFilter.h +++ b/dbms/src/Storages/DeltaMerge/File/DMFilePackFilter.h @@ -21,6 +21,7 @@ #include #include #include +#include #include #include @@ -60,7 +61,8 @@ class DMFilePackFilter bool set_cache_if_miss, const RowKeyRanges & rowkey_ranges, const RSOperatorPtr & filter, - const IdSetPtr & read_packs) + const IdSetPtr & read_packs, + ReadTag read_tag = ReadTag::Internal) { DMFilePackFilter pack_filter( dmfile, @@ -72,7 +74,9 @@ class DMFilePackFilter dm_context.global_context.getFileProvider(), dm_context.global_context.getReadLimiter(), dm_context.scan_context, - dm_context.tracing_id); + dm_context.tracing_id, + dm_context.global_context.getSettingsRef().dt_enable_trim_minmax, + read_tag); return pack_filter.load(); } @@ -86,7 +90,9 @@ class DMFilePackFilter const RowKeyRanges & rowkey_ranges, const RSOperatorPtr & filter, const IdSetPtr & read_packs, - const String & tracing_id) + const String & tracing_id, + bool enable_trim_minmax = false, + ReadTag read_tag = ReadTag::Internal) { DMFilePackFilter pack_filter( dmfile, @@ -98,7 +104,9 @@ class DMFilePackFilter file_provider_, read_limiter_, scan_context, - tracing_id); + tracing_id, + enable_trim_minmax, + read_tag); return pack_filter.load(); } @@ -170,10 +178,14 @@ class DMFilePackFilter const FileProviderPtr & file_provider_, const ReadLimiterPtr & read_limiter_, const ScanContextPtr & scan_context_, - const String & tracing_id) + const String & tracing_id, + bool enable_trim_minmax_, + ReadTag read_tag_ = ReadTag::Internal) : dmfile(dmfile_) , index_cache(index_cache_) , set_cache_if_miss(set_cache_if_miss_) + , enable_trim_minmax(enable_trim_minmax_) + , read_tag(read_tag_) , rowkey_ranges(rowkey_ranges_) , filter(filter_) , read_packs(read_packs_) @@ -196,12 +208,16 @@ class DMFilePackFilter const ScanContextPtr & scan_context); void tryLoadIndex(RSCheckParam & param, ColId col_id); + void tryLoadIndexByRequest(RSCheckParam & param, const RSIndexRequest & request); + TrimMinMaxFallbackReason tryLoadTrimIndex(RSCheckParam & param, ColId col_id, const DateQueryDomain & query_domain); private: DMFilePtr dmfile; MinMaxIndexCachePtr index_cache; bool set_cache_if_miss; + bool enable_trim_minmax = false; + ReadTag read_tag = ReadTag::Internal; RowKeyRanges rowkey_ranges; RSOperatorPtr filter; IdSetPtr read_packs; diff --git a/dbms/src/Storages/DeltaMerge/File/DMFileUtil.cpp b/dbms/src/Storages/DeltaMerge/File/DMFileUtil.cpp index 052660d7fa9..9ad3880910a 100644 --- a/dbms/src/Storages/DeltaMerge/File/DMFileUtil.cpp +++ b/dbms/src/Storages/DeltaMerge/File/DMFileUtil.cpp @@ -50,9 +50,13 @@ String colIndexFileName(const FileNameBase & file_name_base) { return file_name_base + details::INDEX_FILE_SUFFIX; } +String colTrimIndexFileName(const FileNameBase & file_name_base) +{ + return file_name_base + details::TRIM_INDEX_FILE_SUFFIX; +} String colMarkFileName(const FileNameBase & file_name_base) { return file_name_base + details::MARK_FILE_SUFFIX; } -} // namespace DB::DM \ No newline at end of file +} // namespace DB::DM diff --git a/dbms/src/Storages/DeltaMerge/File/DMFileUtil.h b/dbms/src/Storages/DeltaMerge/File/DMFileUtil.h index f5da8efa497..40b7192b7ae 100644 --- a/dbms/src/Storages/DeltaMerge/File/DMFileUtil.h +++ b/dbms/src/Storages/DeltaMerge/File/DMFileUtil.h @@ -28,6 +28,7 @@ inline constexpr static const char * FOLDER_PREFIX_READABLE = "dmf_"; inline constexpr static const char * FOLDER_PREFIX_DROPPED = ".del.dmf_"; inline constexpr static const char * DATA_FILE_SUFFIX = ".dat"; inline constexpr static const char * INDEX_FILE_SUFFIX = ".idx"; +inline constexpr static const char * TRIM_INDEX_FILE_SUFFIX = ".trim.idx"; inline constexpr static const char * MARK_FILE_SUFFIX = ".mrk"; inline String getNGCPath(const String & prefix) @@ -49,6 +50,7 @@ String getNGCPath(const String & parent_path, UInt64 file_id, DMFileStatus statu using FileNameBase = String; String colDataFileName(const FileNameBase & file_name_base); String colIndexFileName(const FileNameBase & file_name_base); +String colTrimIndexFileName(const FileNameBase & file_name_base); String colMarkFileName(const FileNameBase & file_name_base); -} // namespace DB::DM \ No newline at end of file +} // namespace DB::DM diff --git a/dbms/src/Storages/DeltaMerge/File/DMFileWriter.cpp b/dbms/src/Storages/DeltaMerge/File/DMFileWriter.cpp index 2cdc14bbace..de45ec7478c 100644 --- a/dbms/src/Storages/DeltaMerge/File/DMFileWriter.cpp +++ b/dbms/src/Storages/DeltaMerge/File/DMFileWriter.cpp @@ -16,6 +16,7 @@ #include #include #include +#include #include #include @@ -73,6 +74,7 @@ DMFileWriter::DMFileWriter( .avg_size = 0, // ... here ignore some fields with default initializers .indexes = {}, + .trim_minmax_index = {}, #ifndef NDEBUG .additional_data_for_test = {}, #endif @@ -111,10 +113,16 @@ DMFileWriter::WriteBufferFromFileBasePtr DMFileWriter::createMetaFile() void DMFileWriter::addStreams(ColId col_id, DataTypePtr type, bool do_index) { + const auto nested_type = removeNullable(type); + const bool can_trim = options.enable_trim_minmax && dmfile->useMetaV2() && col_id != MutSup::extra_handle_id + && col_id != MutSup::version_col_id && col_id != MutSup::delmark_col_id + && TrimMinMax::isSupportedTemporalType(*nested_type); + auto callback = [&](const IDataType::SubstreamPath & substream_path) { const auto stream_name = DMFile::getFileNameBase(col_id, substream_path); bool substream_can_index = !IDataType::isNullMap(substream_path) && !IDataType::isArraySizes(substream_path) && !IDataType::isStringSizes(substream_path); + const bool do_index_stream = do_index && substream_can_index; auto stream = std::make_unique( dmfile, stream_name, @@ -123,7 +131,13 @@ void DMFileWriter::addStreams(ColId col_id, DataTypePtr type, bool do_index) options.max_compress_block_size, file_provider, write_limiter, - do_index && substream_can_index); + do_index_stream); + if (can_trim && do_index_stream) + { + stream->trim_minmaxes = std::make_shared(*type); + stream->trim_lower = TrimMinMax::defaultLowerBoundPacked(*nested_type); + stream->trim_upper = TrimMinMax::defaultUpperBoundPacked(*nested_type); + } column_streams.emplace(stream_name, std::move(stream)); }; type->enumerateStreams(callback, {}); @@ -207,9 +221,22 @@ void DMFileWriter::writeColumn( // For MutSup::extra_handle_id, we ignore del_mark when add minmax index. // Because we need all rows which satisfy a certain range when place delta index no matter whether the row is a delete row. // For TAG Column, we also ignore del_mark when add minmax index. - stream->minmaxes->addPack( - column, - (col_id == MutSup::extra_handle_id || col_id == MutSup::delmark_col_id) ? nullptr : del_mark); + const ColumnVector * effective_del_mark + = (col_id == MutSup::extra_handle_id || col_id == MutSup::delmark_col_id) ? nullptr : del_mark; + if (stream->trim_minmaxes) + { + TrimMinMax::addOrdinaryAndTrimPack( + *stream->minmaxes, + *stream->trim_minmaxes, + column, + effective_del_mark, + stream->trim_lower, + stream->trim_upper); + } + else + { + stream->minmaxes->addPack(column, effective_del_mark); + } } /// There could already be enough data to compress into the new block. @@ -337,6 +364,35 @@ void DMFileWriter::finalizeColumn(ColId col_id, DataTypePtr type) buffer->next(); } + // Write trim min-max only when the column actually has trimmed outliers. + // Without outliers, ordinary and trim are equivalent for non-NULL values. + if (stream->trim_minmaxes && !is_empty_file && stream->trim_minmaxes->hasAnyTrimmedValue()) + { + dmfile_meta->checkMergedFile(merged_file, file_provider, write_limiter); + + auto fname = colTrimIndexFileName(stream_name); + + auto buffer = ChecksumWriteBufferBuilder::build( + merged_file.buffer, + dmfile->getConfiguration()->getChecksumAlgorithm(), + dmfile->getConfiguration()->getChecksumFrameLength()); + + stream->trim_minmaxes->write(*type, *buffer); + + const size_t trim_index_bytes = buffer->getMaterializedBytes(); + MergedSubFileInfo info{ + fname, + merged_file.file_info.number, + merged_file.file_info.size, + trim_index_bytes}; + dmfile_meta->merged_sub_file_infos[fname] = info; + + merged_file.file_info.size += trim_index_bytes; + buffer->next(); + + col_stat.trim_minmax_index = TrimMinMax::makeDefaultProps(*removeNullable(type), dmfile->getPacks()); + } + // write mark into merged_file_writer if (!is_empty_file) { diff --git a/dbms/src/Storages/DeltaMerge/File/DMFileWriter.h b/dbms/src/Storages/DeltaMerge/File/DMFileWriter.h index ce7693e6dea..df13ce6826c 100644 --- a/dbms/src/Storages/DeltaMerge/File/DMFileWriter.h +++ b/dbms/src/Storages/DeltaMerge/File/DMFileWriter.h @@ -117,6 +117,11 @@ class DMFileWriter MinMaxIndexPtr minmaxes; + // Optional trim min-max builder for MyDate / MyDateTime user columns (V3 only). + MinMaxIndexPtr trim_minmaxes; + UInt64 trim_lower = 0; + UInt64 trim_upper = 0; + MarksInCompressedFilePtr marks; WriteBufferFromFileBasePtr mark_file; @@ -137,16 +142,20 @@ class DMFileWriter CompressionSettings compression_settings; size_t min_compress_block_size{}; size_t max_compress_block_size{}; + /// Controlled by Settings.dt_enable_trim_minmax in production write paths. + bool enable_trim_minmax = false; Options() = default; Options( CompressionSettings compression_settings_, size_t min_compress_block_size_, - size_t max_compress_block_size_) + size_t max_compress_block_size_, + bool enable_trim_minmax_ = false) : compression_settings(compression_settings_) , min_compress_block_size(min_compress_block_size_) , max_compress_block_size(max_compress_block_size_) + , enable_trim_minmax(enable_trim_minmax_) {} Options(const Options & from) = default; diff --git a/dbms/src/Storages/DeltaMerge/File/tests/gtest_dm_meta_version.cpp b/dbms/src/Storages/DeltaMerge/File/tests/gtest_dm_meta_version.cpp index c1f1bcb3487..72ba9403e9a 100644 --- a/dbms/src/Storages/DeltaMerge/File/tests/gtest_dm_meta_version.cpp +++ b/dbms/src/Storages/DeltaMerge/File/tests/gtest_dm_meta_version.cpp @@ -19,6 +19,7 @@ #include #include #include +#include #include #include #include @@ -531,4 +532,80 @@ try } CATCH +TEST_P(LocalDMFile, TrimMinMaxIndexMetaPersistAndDrop) +try +{ + auto dm_file = prepareDMFile(/* file_id= */ 1); + ASSERT_EQ(0, dm_file->metaVersion()); + ASSERT_FALSE(dm_file->getColumnStat(MutSup::extra_handle_id).trim_minmax_index.has_value()); + + dtpb::TrimMinMaxIndexProps props; + props.set_format_version(TrimMinMax::FormatVersionV1); + props.set_lower_bound(TrimMinMax::encodeBound(100)); + props.set_upper_bound(TrimMinMax::encodeBound(200)); + props.set_pack_count(dm_file->getPacks()); + + { + auto iw = DMFileV3IncrementWriter::create(DMFileV3IncrementWriter::Options{ + .dm_file = dm_file, + .file_provider = file_provider_maybe_encrypted, + .write_limiter = db_context->getWriteLimiter(), + .path_pool = path_pool, + .disagg_ctx = db_context->getSharedContextDisagg(), + }); + dm_file->meta->getColumnStats()[MutSup::extra_handle_id].trim_minmax_index = props; + ASSERT_EQ(1, dm_file->meta->bumpMetaVersion({})); + iw->finalize(); + } + + // New Reader restores field 105 from MetaV2. + dm_file = DMFile::restore( + file_provider_maybe_encrypted, + 1, + 1, + parent_path, + DMFileMeta::ReadMode::all(), + /* meta_version= */ 1); + ASSERT_TRUE(dm_file->getColumnStat(MutSup::extra_handle_id).trim_minmax_index.has_value()); + EXPECT_EQ(dm_file->getColumnStat(MutSup::extra_handle_id).trim_minmax_index->pack_count(), dm_file->getPacks()); + EXPECT_EQ( + dm_file->getColumnStat(MutSup::extra_handle_id).trim_minmax_index->format_version(), + TrimMinMax::FormatVersionV1); + + // Simulate an old node rewriting ColumnStat without field 105. + { + auto iw = DMFileV3IncrementWriter::create(DMFileV3IncrementWriter::Options{ + .dm_file = dm_file, + .file_provider = file_provider_maybe_encrypted, + .write_limiter = db_context->getWriteLimiter(), + .path_pool = path_pool, + .disagg_ctx = db_context->getSharedContextDisagg(), + }); + dm_file->meta->getColumnStats()[MutSup::extra_handle_id].trim_minmax_index.reset(); + ASSERT_EQ(2, dm_file->meta->bumpMetaVersion({})); + iw->finalize(); + } + + dm_file = DMFile::restore( + file_provider_maybe_encrypted, + 1, + 1, + parent_path, + DMFileMeta::ReadMode::all(), + /* meta_version= */ 2); + ASSERT_FALSE(dm_file->getColumnStat(MutSup::extra_handle_id).trim_minmax_index.has_value()); + + // Without meta, Reader selection must fall back even if a trim fname is presented. + auto reason = TrimMinMax::trySelectTrimMeta( + /*read_enabled*/ true, + dm_file->getColumnStat(MutSup::extra_handle_id).trim_minmax_index, + *dm_file->getColumnStat(MutSup::extra_handle_id).type, + dm_file->getPacks(), + /*merged*/ {}, + colTrimIndexFileName(DB::toString(MutSup::extra_handle_id)), + nullptr); + EXPECT_EQ(reason, TrimMinMaxFallbackReason::NoMeta); +} +CATCH + } // namespace DB::DM::tests diff --git a/dbms/src/Storages/DeltaMerge/Filter/DateQueryDomain.cpp b/dbms/src/Storages/DeltaMerge/Filter/DateQueryDomain.cpp new file mode 100644 index 00000000000..612ef297cd2 --- /dev/null +++ b/dbms/src/Storages/DeltaMerge/Filter/DateQueryDomain.cpp @@ -0,0 +1,393 @@ +// Copyright 2026 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +namespace DB::DM +{ +namespace +{ +bool tryGetUInt64(const Field & f, UInt64 & out) +{ + if (f.isNull()) + return false; + if (f.getType() == Field::Types::UInt64) + { + out = f.get(); + return true; + } + if (f.getType() == Field::Types::Int64) + { + const auto v = f.get(); + if (v < 0) + return false; + out = static_cast(v); + return true; + } + return false; +} + +bool inHalfOpenRange(UInt64 value, UInt64 lower, UInt64 upper) +{ + return value >= lower && value < upper; +} + +void flattenTopLevelAnd(const RSOperatorPtr & op, RSOperators & out) +{ + if (!op) + return; + + // Iterative flatten of top-level And nesting to avoid deep call stacks on + // left-associated And(And(And(...))) chains. + RSOperators stack{op}; + while (!stack.empty()) + { + auto cur = stack.back(); + stack.pop_back(); + if (auto and_op = std::dynamic_pointer_cast(cur)) + { + const auto & children = and_op->getChildren(); + // Push in reverse so that left-to-right child order is preserved in `out`. + for (const auto & it : std::ranges::reverse_view(children)) + stack.push_back(it); + continue; + } + out.push_back(cur); + } +} + +enum class BoundSide : UInt8 +{ + LowerInclusive, + LowerExclusive, + UpperInclusive, + UpperExclusive, +}; + +struct TemporalBound +{ + BoundSide side; + Field value; + Attr attr; +}; + +std::optional tryAsTemporalRangeBound(const RSOperatorPtr & op) +{ + auto take = [&](const Attr & attr, const Field & value, BoundSide side) -> std::optional { + if (!attr.type || !TrimMinMax::isSupportedTemporalType(*attr.type)) + return std::nullopt; + return TemporalBound{.side = side, .value = value, .attr = attr}; + }; + + if (auto ge = std::dynamic_pointer_cast(op)) + return take(ge->getAttr(), ge->getValue(), BoundSide::LowerInclusive); + if (auto gt = std::dynamic_pointer_cast(op)) + return take(gt->getAttr(), gt->getValue(), BoundSide::LowerExclusive); + if (auto le = std::dynamic_pointer_cast(op)) + return take(le->getAttr(), le->getValue(), BoundSide::UpperInclusive); + if (auto lt = std::dynamic_pointer_cast(op)) + return take(lt->getAttr(), lt->getValue(), BoundSide::UpperExclusive); + return std::nullopt; +} + +struct BoundAccumulator +{ + Attr attr; + std::optional lower; + bool lower_inclusive = true; + std::optional upper; + bool upper_inclusive = true; + bool failed = false; + RSOperators originals; +}; + +bool applyBound(BoundAccumulator & acc, const TemporalBound & bound) +{ + UInt64 nv = 0; + if (!tryGetUInt64(bound.value, nv)) + return false; + + switch (bound.side) + { + case BoundSide::LowerInclusive: + case BoundSide::LowerExclusive: + { + const bool inclusive = bound.side == BoundSide::LowerInclusive; + if (!acc.lower) + { + acc.lower = bound.value; + acc.lower_inclusive = inclusive; + return true; + } + UInt64 ov = 0; + if (!tryGetUInt64(*acc.lower, ov)) + return false; + // Stronger lower: larger value; on tie prefer exclusive. + if (nv > ov || (nv == ov && !inclusive && acc.lower_inclusive)) + { + acc.lower = bound.value; + acc.lower_inclusive = inclusive; + } + break; + } + case BoundSide::UpperInclusive: + case BoundSide::UpperExclusive: + { + const bool inclusive = bound.side == BoundSide::UpperInclusive; + if (!acc.upper) + { + acc.upper = bound.value; + acc.upper_inclusive = inclusive; + return true; + } + UInt64 ov = 0; + if (!tryGetUInt64(*acc.upper, ov)) + return false; + // Stronger upper: smaller value; on tie prefer exclusive. + if (nv < ov || (nv == ov && !inclusive && acc.upper_inclusive)) + { + acc.upper = bound.value; + acc.upper_inclusive = inclusive; + } + break; + } + } + return true; +} +} // namespace + +bool DateQueryDomain::isTrimEligible(UInt64 stored_lower, UInt64 stored_upper) const +{ + auto endpoint_in_e = [&](const Field & f) { + UInt64 v = 0; + return tryGetUInt64(f, v) && inHalfOpenRange(v, stored_lower, stored_upper); + }; + + switch (predicate_class) + { + case TrimPredicateClass::EqualityOrInOrBounded: + { + if (!values.empty()) + { + for (const auto & v : values) + { + if (!endpoint_in_e(v)) + return false; + } + return true; + } + // Bounded range described by lower/upper endpoints: require Q βŠ† E by requiring + // both finite endpoints to lie in E (conservative for inclusive bounds). + if (!lower || !upper) + return false; + return endpoint_in_e(*lower) && endpoint_in_e(*upper); + } + case TrimPredicateClass::LowerBounded: + return lower.has_value() && endpoint_in_e(*lower); + case TrimPredicateClass::UpperBounded: + return upper.has_value() && endpoint_in_e(*upper); + } + return false; +} + +RSResults applyTrimRoughCheckCorrection( + const RSResults & raw, + size_t start_pack, + const MinMaxIndex & trim_minmax, + TrimPredicateClass predicate_class, + bool record_metrics) +{ + RSResults results = raw; + UInt64 none_count = 0; + UInt64 some_count = 0; + UInt64 all_count = 0; + UInt64 all_null_count = 0; + UInt64 none_to_some_count = 0; + UInt64 all_to_some_count = 0; + for (size_t i = 0; i < results.size(); ++i) + { + const size_t pack_id = start_pack + i; + const bool has_trimmed_low = trim_minmax.hasTrimmedLow(pack_id); + const bool has_trimmed_high = trim_minmax.hasTrimmedHigh(pack_id); + + // Map pack-level low/high outlier flags to "must match" / "must not match" + // under the current predicate class. Outliers live in D-E and are invisible + // to the trim min-max, so raw None/All may need correction below. + bool trimmed_match_exists = false; + bool trimmed_nonmatch_exists = false; + switch (predicate_class) + { + case TrimPredicateClass::EqualityOrInOrBounded: + { + // Q βŠ† E: any low/high trimmed value is outside Q, so it never matches. + // It can only invalidate an All (pack still has a non-matching outlier). + trimmed_match_exists = false; + trimmed_nonmatch_exists = has_trimmed_low || has_trimmed_high; + break; + } + case TrimPredicateClass::LowerBounded: + { + // col >= T / col > T with T in E: a high outlier always matches, + // a low outlier never matches. + trimmed_match_exists = has_trimmed_high; + trimmed_nonmatch_exists = has_trimmed_low; + break; + } + case TrimPredicateClass::UpperBounded: + { + // col <= T / col < T with T in E: a low outlier always matches, + // a high outlier never matches. + trimmed_match_exists = has_trimmed_low; + trimmed_nonmatch_exists = has_trimmed_high; + break; + } + } + + auto & r = results[i]; + if (trimmed_match_exists) + { + if (r == RSResult::None) + { + r = RSResult::Some; + if (record_metrics) + ++none_to_some_count; + } + else if (r == RSResult::NoneNull) + { + r = RSResult::SomeNull; + if (record_metrics) + ++none_to_some_count; + } + } + if (trimmed_nonmatch_exists) + { + if (r == RSResult::All) + { + r = RSResult::Some; + if (record_metrics) + ++all_to_some_count; + } + else if (r == RSResult::AllNull) + { + r = RSResult::SomeNull; + if (record_metrics) + ++all_to_some_count; + } + } + + if (record_metrics) + { + if (r == RSResult::None || r == RSResult::NoneNull) + ++none_count; + else if (r == RSResult::Some || r == RSResult::SomeNull) + ++some_count; + else if (r == RSResult::All) + ++all_count; + else if (r == RSResult::AllNull) + ++all_null_count; + } + } + + if (record_metrics) + { + if (none_count != 0) + GET_METRIC(tiflash_storage_trim_minmax_rough_check_pack_count, result_none).Increment(none_count); + if (some_count != 0) + GET_METRIC(tiflash_storage_trim_minmax_rough_check_pack_count, result_some).Increment(some_count); + if (all_count != 0) + GET_METRIC(tiflash_storage_trim_minmax_rough_check_pack_count, result_all).Increment(all_count); + if (all_null_count != 0) + GET_METRIC(tiflash_storage_trim_minmax_rough_check_pack_count, result_all_null).Increment(all_null_count); + if (none_to_some_count != 0) + GET_METRIC(tiflash_storage_trim_minmax_correction_pack_count, type_none_to_some) + .Increment(none_to_some_count); + if (all_to_some_count != 0) + GET_METRIC(tiflash_storage_trim_minmax_correction_pack_count, type_all_to_some) + .Increment(all_to_some_count); + } + return results; +} + +RSOperatorPtr normalizeTemporalRangesForTrim(const RSOperatorPtr & op) +{ + if (!op) + return op; + + RSOperators leaves; + flattenTopLevelAnd(op, leaves); + + std::unordered_map bounds; + RSOperators kept; + kept.reserve(leaves.size()); + + for (const auto & leaf : leaves) + { + if (auto bound = tryAsTemporalRangeBound(leaf)) + { + auto & acc = bounds[bound->attr.col_id]; + if (acc.originals.empty()) + acc.attr = bound->attr; + acc.originals.push_back(leaf); + if (!applyBound(acc, *bound)) + acc.failed = true; + continue; + } + kept.push_back(leaf); + } + + for (auto & [col_id, acc] : bounds) + { + (void)col_id; + // Any unparseable bound (or no successful bound) abandons DateRange merge for the column. + if (acc.failed || (!acc.lower && !acc.upper)) + { + kept.insert(kept.end(), acc.originals.begin(), acc.originals.end()); + continue; + } + + DateQueryDomain domain; + domain.lower = acc.lower; + domain.lower_inclusive = acc.lower_inclusive; + domain.upper = acc.upper; + domain.upper_inclusive = acc.upper_inclusive; + if (acc.lower && acc.upper) + domain.predicate_class = TrimPredicateClass::EqualityOrInOrBounded; + else if (acc.lower) + domain.predicate_class = TrimPredicateClass::LowerBounded; + else + domain.predicate_class = TrimPredicateClass::UpperBounded; + kept.push_back(createDateRange(acc.attr, std::move(domain))); + } + + if (kept.empty()) + return EMPTY_RS_OPERATOR; + if (kept.size() == 1) + return kept[0]; + return createAnd(kept); +} + +} // namespace DB::DM diff --git a/dbms/src/Storages/DeltaMerge/Filter/DateQueryDomain.h b/dbms/src/Storages/DeltaMerge/Filter/DateQueryDomain.h new file mode 100644 index 00000000000..92b3c18f8b5 --- /dev/null +++ b/dbms/src/Storages/DeltaMerge/Filter/DateQueryDomain.h @@ -0,0 +1,87 @@ +// Copyright 2026 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +#include +#include +#include +#include +#include +#include + +#include +#include + +namespace DB::DM +{ + +enum class TrimPredicateClass : UInt8 +{ + /// equality / IN / bounded range with Q βŠ† E. + EqualityOrInOrBounded = 0, + /// col >= L or col > L, with L ∈ E. + LowerBounded = 1, + /// col <= U or col < U, with U ∈ E. + UpperBounded = 2, +}; + +enum class RSIndexKind : UInt8 +{ + Normal = 0, + PreferTrim = 1, +}; + +/// Normalized non-NULL match set for a temporal column predicate. +/// Bounds are already timezone-converted packed values when applicable. +struct DateQueryDomain +{ + TrimPredicateClass predicate_class = TrimPredicateClass::EqualityOrInOrBounded; + + /// For EqualityOrInOrBounded: all candidate values (points) that must lie in E. + /// For ranges, also used to carry the finite endpoints that define Q. + std::vector values; + + /// Optional range endpoints. Inclusive/exclusive flags describe Q. + std::optional lower; + bool lower_inclusive = true; + std::optional upper; + bool upper_inclusive = true; + + /// True when every value / finite endpoint that must belong to E is inside [stored_lower, stored_upper). + bool isTrimEligible(UInt64 stored_lower, UInt64 stored_upper) const; +}; + +struct RSIndexRequest +{ + ColId col_id = 0; + RSIndexKind preferred_kind = RSIndexKind::Normal; + std::optional query_domain; +}; + +using RSIndexRequests = std::vector; + +/// Apply conservative None/All downgrades using per-pack trim flags. +RSResults applyTrimRoughCheckCorrection( + const RSResults & raw, + size_t start_pack, + const MinMaxIndex & trim_minmax, + TrimPredicateClass predicate_class, + bool record_metrics = false); + +/// Flatten top-level AND and merge temporal GE/GT/LE/LT into DateRange PreferTrim ops. +/// Does not rewrite children under OR / NOT. Equal / In are left unchanged. +RSOperatorPtr normalizeTemporalRangesForTrim(const RSOperatorPtr & op); + +} // namespace DB::DM diff --git a/dbms/src/Storages/DeltaMerge/Filter/DateRange.h b/dbms/src/Storages/DeltaMerge/Filter/DateRange.h new file mode 100644 index 00000000000..30e3834a465 --- /dev/null +++ b/dbms/src/Storages/DeltaMerge/Filter/DateRange.h @@ -0,0 +1,198 @@ +// Copyright 2026 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +#include +#include +#include + +#include + +namespace DB::DM +{ + +/// Normalized temporal range used only for Rough Set filtering. +/// Row-level filters remain the original DAG expressions. +class DateRange : public RSOperator +{ +public: + DateRange(const Attr & attr_, DateQueryDomain domain_) + : attr(attr_) + , domain(std::move(domain_)) + {} + + String name() override { return "date_range"; } + + ColIds getColumnIDs() override { return {attr.col_id}; } + + RSIndexRequests getIndexRequests() override + { + return {RSIndexRequest{ + .col_id = attr.col_id, + .preferred_kind = RSIndexKind::PreferTrim, + .query_domain = domain, + }}; + } + + String toDebugString() override + { + return fmt::format( + R"({{"op":"{}","col":"{}","class":"{}","lower":"{}","lower_inclusive":{},"upper":"{}","upper_inclusive":{}}})", + name(), + attr.col_name, + magic_enum::enum_name(domain.predicate_class), + domain.lower ? applyVisitor(FieldVisitorToDebugString(), *domain.lower) : "", + domain.lower_inclusive, + domain.upper ? applyVisitor(FieldVisitorToDebugString(), *domain.upper) : "", + domain.upper_inclusive); + } + + Poco::JSON::Object::Ptr toJSONObject() override + { + Poco::JSON::Object::Ptr obj = new Poco::JSON::Object(); + obj->set("op", name()); + obj->set("col", attr.col_name); + obj->set("class", String(magic_enum::enum_name(domain.predicate_class))); + if (domain.lower) + { + obj->set("lower", applyVisitor(FieldVisitorToDebugString(), *domain.lower)); + obj->set("lower_inclusive", domain.lower_inclusive); + } + if (domain.upper) + { + obj->set("upper", applyVisitor(FieldVisitorToDebugString(), *domain.upper)); + obj->set("upper_inclusive", domain.upper_inclusive); + } + return obj; + } + + RSResults roughCheck(size_t start_pack, size_t pack_count, const RSCheckParam & param) override + { + if (auto trim = getTrimRSIndex(param, attr, domain)) + { + auto raw = checkRange(start_pack, pack_count, trim->type, trim->minmax); + return applyTrimRoughCheckCorrection( + raw, + start_pack, + *trim->minmax, + domain.predicate_class, + param.record_trim_metrics); + } + + if (auto rs_index = getRSIndex(param, attr)) + return checkRange(start_pack, pack_count, rs_index->type, rs_index->minmax); + + return RSResults(pack_count, RSResult::Some); + } + + ColumnRangePtr buildSets(const google::protobuf::RepeatedPtrField & index_infos) override + { + IntegerSetPtr set; + if (domain.lower && domain.upper) + { + auto lower_set + = IntegerSet::createGreaterRangeSet(attr.type, *domain.lower, /*not_included=*/!domain.lower_inclusive); + auto upper_set + = IntegerSet::createLessRangeSet(attr.type, *domain.upper, /*not_included=*/!domain.upper_inclusive); + if (lower_set && upper_set) + set = lower_set->intersectWith(upper_set); + } + else if (domain.lower) + { + set = IntegerSet::createGreaterRangeSet(attr.type, *domain.lower, /*not_included=*/!domain.lower_inclusive); + } + else if (domain.upper) + { + set = IntegerSet::createLessRangeSet(attr.type, *domain.upper, /*not_included=*/!domain.upper_inclusive); + } + + if (!set) + return UnsupportedColumnRange::create(); + + auto iter = std::find_if(index_infos.begin(), index_infos.end(), [&](const auto & info) { + return info.index_type() == tipb::ColumnarIndexType::TypeInverted + && info.inverted_query_info().column_id() == attr.col_id; + }); + if (iter != index_infos.end()) + return SingleColumnRange::create( + iter->inverted_query_info().column_id(), + iter->inverted_query_info().index_id(), + set); + return UnsupportedColumnRange::create(); + } + + const DateQueryDomain & getDomain() const { return domain; } + const Attr & getAttr() const { return attr; } + +private: + /// Pack-level rough check of the temporal range described by `domain` against `minmax`. + RSResults checkRange(size_t start_pack, size_t pack_count, const DataTypePtr & type, const MinMaxIndexPtr & minmax) + const + { + // Empty domain must not advertise All (would skip row-level filtering incorrectly). + if (!domain.lower && !domain.upper) + return RSResults(pack_count, RSResult::Some); + + RSResults results(pack_count, RSResult::All); + if (domain.lower) + { + RSResults lower_res = domain.lower_inclusive + ? minmax->checkCmp(start_pack, pack_count, *domain.lower, type) + : minmax->checkCmp(start_pack, pack_count, *domain.lower, type); + std::transform( + results.begin(), + results.end(), + lower_res.begin(), + results.begin(), + [](const auto a, const auto b) { return a && b; }); + } + if (domain.upper) + { + RSResults upper_res; + if (domain.upper_inclusive) + { + // col <= U <=> !(col > U) + upper_res = minmax->checkCmp(start_pack, pack_count, *domain.upper, type); + } + else + { + // col < U <=> !(col >= U) + upper_res + = minmax->checkCmp(start_pack, pack_count, *domain.upper, type); + } + for (size_t i = 0; i < pack_count; ++i) + { + // MinMax only exposes greater/greater-equal checks, so upper bounds are + // evaluated as the negation of those results (col <= U <=> !(col > U)). + if (minmax->hasValue(start_pack + i)) + upper_res[i] = !upper_res[i]; + // else: leave None for empty packs (has_value=false). + // Negating empty-pack None into All breaks trim outlier corrections. + } + std::transform( + results.begin(), + results.end(), + upper_res.begin(), + results.begin(), + [](const auto a, const auto b) { return a && b; }); + } + return results; + } + + Attr attr; + DateQueryDomain domain; +}; + +} // namespace DB::DM diff --git a/dbms/src/Storages/DeltaMerge/Filter/Equal.h b/dbms/src/Storages/DeltaMerge/Filter/Equal.h index 2422098903f..8f2b63a4279 100644 --- a/dbms/src/Storages/DeltaMerge/Filter/Equal.h +++ b/dbms/src/Storages/DeltaMerge/Filter/Equal.h @@ -14,8 +14,10 @@ #pragma once +#include #include #include +#include namespace DB::DM { @@ -29,8 +31,40 @@ class Equal : public ColCmpVal String name() override { return "equal"; } + DateQueryDomain makeQueryDomain() const + { + DateQueryDomain domain; + domain.predicate_class = TrimPredicateClass::EqualityOrInOrBounded; + domain.values = {value}; + return domain; + } + + RSIndexRequests getIndexRequests() override + { + // attr.type can be empty when column id is missing from table defines. + if (attr.type && TrimMinMax::isSupportedTemporalType(*attr.type)) + { + return {RSIndexRequest{ + .col_id = attr.col_id, + .preferred_kind = RSIndexKind::PreferTrim, + .query_domain = makeQueryDomain(), + }}; + } + return RSOperator::getIndexRequests(); + } + RSResults roughCheck(size_t start_pack, size_t pack_count, const RSCheckParam & param) override { + if (auto trim = getTrimRSIndex(param, attr, makeQueryDomain())) + { + auto raw = trim->minmax->checkCmp(start_pack, pack_count, value, trim->type); + return applyTrimRoughCheckCorrection( + raw, + start_pack, + *trim->minmax, + TrimPredicateClass::EqualityOrInOrBounded, + param.record_trim_metrics); + } return minMaxCheckCmp(start_pack, pack_count, param, attr, value); } diff --git a/dbms/src/Storages/DeltaMerge/Filter/In.h b/dbms/src/Storages/DeltaMerge/Filter/In.h index 797623175a0..b968c946c3b 100644 --- a/dbms/src/Storages/DeltaMerge/Filter/In.h +++ b/dbms/src/Storages/DeltaMerge/Filter/In.h @@ -14,7 +14,9 @@ #pragma once +#include #include +#include namespace DB::DM { @@ -34,6 +36,32 @@ class In : public RSOperator ColIds getColumnIDs() override { return {attr.col_id}; } + DateQueryDomain makeQueryDomain() const + { + DateQueryDomain domain; + domain.predicate_class = TrimPredicateClass::EqualityOrInOrBounded; + for (const auto & v : values) + { + if (!v.isNull()) + domain.values.push_back(v); + } + return domain; + } + + RSIndexRequests getIndexRequests() override + { + // attr.type can be empty when column id is missing from table defines. + if (attr.type && TrimMinMax::isSupportedTemporalType(*attr.type)) + { + return {RSIndexRequest{ + .col_id = attr.col_id, + .preferred_kind = RSIndexKind::PreferTrim, + .query_domain = makeQueryDomain(), + }}; + } + return RSOperator::getIndexRequests(); + } + String toDebugString() override { FmtBuffer buf; @@ -69,6 +97,18 @@ class In : public RSOperator // So return none directly. if (values.empty()) return RSResults(pack_count, RSResult::None); + + if (auto trim = getTrimRSIndex(param, attr, makeQueryDomain())) + { + auto raw = trim->minmax->checkIn(start_pack, pack_count, values, trim->type); + return applyTrimRoughCheckCorrection( + raw, + start_pack, + *trim->minmax, + TrimPredicateClass::EqualityOrInOrBounded, + param.record_trim_metrics); + } + auto rs_index = getRSIndex(param, attr); return rs_index ? rs_index->minmax->checkIn(start_pack, pack_count, values, rs_index->type) : RSResults(pack_count, RSResult::Some); diff --git a/dbms/src/Storages/DeltaMerge/Filter/Not.h b/dbms/src/Storages/DeltaMerge/Filter/Not.h index 8fd5da56319..2d10825141e 100644 --- a/dbms/src/Storages/DeltaMerge/Filter/Not.h +++ b/dbms/src/Storages/DeltaMerge/Filter/Not.h @@ -28,9 +28,29 @@ class Not : public LogicalOp String name() override { return "not"; } + /// Design excludes NOT / NOT IN from trim support. Do not forward PreferTrim from children. + RSIndexRequests getIndexRequests() override + { + auto reqs = LogicalOp::getIndexRequests(); + for (auto & req : reqs) + { + if (req.preferred_kind == RSIndexKind::PreferTrim) + { + req.preferred_kind = RSIndexKind::Normal; + req.query_domain.reset(); + } + } + return reqs; + } + RSResults roughCheck(size_t start_pack, size_t pack_count, const RSCheckParam & param) override { - auto results = children[0]->roughCheck(start_pack, pack_count, param); + // Clear trim indexes so the child cannot reuse a trim loaded by a sibling PreferTrim + // leaf (e.g. Equal AND Not(In(...))). Empty trim packs skip CheckIn's NULL handling + // and would turn In(...NULL...) into None, then !None => All and skip row filters. + RSCheckParam child_param = param; + child_param.trim_indexes.clear(); + auto results = children[0]->roughCheck(start_pack, pack_count, child_param); std::transform(results.begin(), results.end(), results.begin(), [](const auto result) { return !result; }); return results; } diff --git a/dbms/src/Storages/DeltaMerge/Filter/PushDownExecutor.cpp b/dbms/src/Storages/DeltaMerge/Filter/PushDownExecutor.cpp index 2288a87633c..89d83c2b0f4 100644 --- a/dbms/src/Storages/DeltaMerge/Filter/PushDownExecutor.cpp +++ b/dbms/src/Storages/DeltaMerge/Filter/PushDownExecutor.cpp @@ -207,7 +207,8 @@ PushDownExecutorPtr PushDownExecutor::build( columns_to_read_info, table_column_defines, context.getSettingsRef().dt_enable_rough_set_filter, - tracing_logger); + tracing_logger, + context.getSettingsRef().dt_enable_trim_minmax); // build column_range const auto column_range = rs_operator && !used_indexes.empty() ? rs_operator->buildSets(used_indexes) : nullptr; // build ann_query_info diff --git a/dbms/src/Storages/DeltaMerge/Filter/RSOperator.cpp b/dbms/src/Storages/DeltaMerge/Filter/RSOperator.cpp index adb9eccbdd9..7efa64a642c 100644 --- a/dbms/src/Storages/DeltaMerge/Filter/RSOperator.cpp +++ b/dbms/src/Storages/DeltaMerge/Filter/RSOperator.cpp @@ -15,6 +15,7 @@ #include #include #include +#include #include #include #include @@ -51,6 +52,7 @@ RSOperatorPtr createNotEqual(const Attr & attr, const Field & value) RSOperatorPtr createOr(const RSOperators & children) { return std::make_shared(children); } RSOperatorPtr createIsNull(const Attr & attr) { return std::make_shared(attr);} RSOperatorPtr createUnsupported(const String & reason) { return std::make_shared(reason); } +RSOperatorPtr createDateRange(const Attr & attr, DateQueryDomain domain) { return std::make_shared(attr, std::move(domain)); } // clang-format on RSOperatorPtr RSOperator::build( @@ -58,7 +60,8 @@ RSOperatorPtr RSOperator::build( const TiDB::ColumnInfos & scan_column_infos, const ColumnDefines & table_column_defines, bool enable_rs_filter, - const LoggerPtr & tracing_logger) + const LoggerPtr & tracing_logger, + bool enable_trim_minmax) { RUNTIME_CHECK(dag_query != nullptr); // build rough set operator @@ -96,7 +99,12 @@ RSOperatorPtr RSOperator::build( column_id_to_attr[col_info.id] = attr; } - auto rs_operator = FilterParser::parseDAGQuery(*dag_query, scan_column_infos, column_id_to_attr, tracing_logger); + auto rs_operator = FilterParser::parseDAGQuery( + *dag_query, + scan_column_infos, + column_id_to_attr, + tracing_logger, + enable_trim_minmax); if (likely(rs_operator != DM::EMPTY_RS_OPERATOR)) LOG_DEBUG(tracing_logger, "Rough set filter: {}", rs_operator->toDebugString()); diff --git a/dbms/src/Storages/DeltaMerge/Filter/RSOperator.h b/dbms/src/Storages/DeltaMerge/Filter/RSOperator.h index 0fd5c2fcd6a..71ae5a78cc1 100644 --- a/dbms/src/Storages/DeltaMerge/Filter/RSOperator.h +++ b/dbms/src/Storages/DeltaMerge/Filter/RSOperator.h @@ -22,6 +22,7 @@ #include #include #include +#include #include #include #include @@ -41,7 +42,12 @@ inline static const RSOperatorPtr EMPTY_RS_OPERATOR{}; struct RSCheckParam { + /// Ordinary min-max indexes (existing field name kept for compatibility). ColumnIndexes indexes; + /// Trim min-max indexes selected for PreferTrim requests. + TrimColumnIndexes trim_indexes; + /// Prometheus trim metrics are recorded only for the query read pass, avoiding duplicate counts from MVCC/LM passes. + bool record_trim_metrics = false; }; class RSOperator @@ -60,6 +66,15 @@ class RSOperator virtual ColIds getColumnIDs() = 0; + /// Index load requests. Default maps getColumnIDs() to Normal requests. + virtual RSIndexRequests getIndexRequests() + { + RSIndexRequests reqs; + for (const auto id : getColumnIDs()) + reqs.push_back(RSIndexRequest{.col_id = id, .preferred_kind = RSIndexKind::Normal, .query_domain = {}}); + return reqs; + } + virtual ColumnRangePtr buildSets(const google::protobuf::RepeatedPtrField & index_infos) = 0; @@ -68,7 +83,8 @@ class RSOperator const TiDB::ColumnInfos & scan_column_infos, const ColumnDefines & table_column_defines, bool enable_rs_filter, - const LoggerPtr & tracing_logger); + const LoggerPtr & tracing_logger, + bool enable_trim_minmax = false); }; class ColCmpVal : public RSOperator @@ -83,6 +99,9 @@ class ColCmpVal : public RSOperator , value(value_) {} + const Attr & getAttr() const { return attr; } + const Field & getValue() const { return value; } + ColIds getColumnIDs() override { return {attr.col_id}; } String toDebugString() override @@ -114,6 +133,8 @@ class LogicalOp : public RSOperator : children(children_) {} + const RSOperators & getChildren() const { return children; } + ColIds getColumnIDs() override { ColIds col_ids; @@ -125,6 +146,17 @@ class LogicalOp : public RSOperator return col_ids; } + RSIndexRequests getIndexRequests() override + { + RSIndexRequests reqs; + for (const auto & child : children) + { + auto child_reqs = child->getIndexRequests(); + reqs.insert(reqs.end(), child_reqs.begin(), child_reqs.end()); + } + return reqs; + } + String toDebugString() override { FmtBuffer buf; @@ -153,6 +185,8 @@ class LogicalOp : public RSOperator inline std::optional getRSIndex(const RSCheckParam & param, const Attr & attr) { + if (!attr.type) + return std::nullopt; auto it = param.indexes.find(attr.col_id); if (it != param.indexes.end() && it->second.type->equals(*attr.type)) { @@ -161,6 +195,24 @@ inline std::optional getRSIndex(const RSCheckParam & param, const Attr return std::nullopt; } +/// Return the column's trim index only when `query_domain` is trim-eligible for the +/// stored E. Trim indexes are cached per column, so callers must re-check the +/// current predicate's domain before using trim. +inline std::optional getTrimRSIndex( + const RSCheckParam & param, + const Attr & attr, + const DateQueryDomain & query_domain) +{ + if (!attr.type) + return std::nullopt; + auto it = param.trim_indexes.find(attr.col_id); + if (it == param.trim_indexes.end() || !it->second.type->equals(*attr.type)) + return std::nullopt; + if (!query_domain.isTrimEligible(it->second.meta.lower_bound, it->second.meta.upper_bound)) + return std::nullopt; + return it->second; +} + template RSResults minMaxCheckCmp( size_t start_pack, @@ -195,4 +247,6 @@ RSOperatorPtr createIsNull(const Attr & attr); // RSOperatorPtr createUnsupported(const String & reason); +RSOperatorPtr createDateRange(const Attr & attr, DateQueryDomain domain); + } // namespace DB::DM diff --git a/dbms/src/Storages/DeltaMerge/FilterParser/FilterParser.cpp b/dbms/src/Storages/DeltaMerge/FilterParser/FilterParser.cpp index f2c797f72f3..49fdab6d97a 100644 --- a/dbms/src/Storages/DeltaMerge/FilterParser/FilterParser.cpp +++ b/dbms/src/Storages/DeltaMerge/FilterParser/FilterParser.cpp @@ -16,6 +16,7 @@ #include #include #include +#include #include #include #include @@ -166,6 +167,16 @@ inline RSOperatorPtr parseTiCompareExpr( // auto col_id = getColumnIDForColumnExpr(child, scan_column_infos); attr = id_to_attr.at(col_id); + // Column ID may be absent from table defines (e.g. function ColumnRef). + // Attr.type is then empty; creating Equal/In would crash in getIndexRequests + // on *attr.type. Convert to Unsupported so rough check stays conservative Some. + if (unlikely(!attr.type)) + { + return createUnsupported(fmt::format( + "ColumnRef with unknown column id is not supported, sig={} col_id={}", + tipb::ScalarFuncSig_Name(expr.sig()), + col_id)); + } } else if (isLiteralExpr(child)) { @@ -366,7 +377,8 @@ RSOperatorPtr FilterParser::parseDAGQuery( const DAGQueryInfo & dag_info, const TiDB::ColumnInfos & scan_column_infos, const ColumnIDToAttrMap & id_to_attr, - const LoggerPtr & log) + const LoggerPtr & log, + bool enable_trim_minmax) { /// By default, multiple conditions with operator "and" RSOperators children; @@ -382,10 +394,11 @@ RSOperatorPtr FilterParser::parseDAGQuery( if (children.empty()) return EMPTY_RS_OPERATOR; - else if (children.size() == 1) - return children[0]; - else - return createAnd(children); + + RSOperatorPtr op = children.size() == 1 ? children[0] : createAnd(children); + if (enable_trim_minmax) + return normalizeTemporalRangesForTrim(op); + return op; } RSOperatorPtr FilterParser::parseRFInExpr( diff --git a/dbms/src/Storages/DeltaMerge/FilterParser/FilterParser.h b/dbms/src/Storages/DeltaMerge/FilterParser/FilterParser.h index a160baf2c6d..d773355d3aa 100644 --- a/dbms/src/Storages/DeltaMerge/FilterParser/FilterParser.h +++ b/dbms/src/Storages/DeltaMerge/FilterParser/FilterParser.h @@ -41,7 +41,8 @@ class FilterParser const DAGQueryInfo & dag_info, const TiDB::ColumnInfos & scan_column_infos, const ColumnIDToAttrMap & id_to_attr, - const LoggerPtr & log); + const LoggerPtr & log, + bool enable_trim_minmax = false); // only for runtime filter in predicate static RSOperatorPtr parseRFInExpr( diff --git a/dbms/src/Storages/DeltaMerge/Index/MinMaxIndex.cpp b/dbms/src/Storages/DeltaMerge/Index/MinMaxIndex.cpp index a10d26bb035..e3d298328e8 100644 --- a/dbms/src/Storages/DeltaMerge/Index/MinMaxIndex.cpp +++ b/dbms/src/Storages/DeltaMerge/Index/MinMaxIndex.cpp @@ -15,6 +15,7 @@ #include #include #include +#include #include #include #include @@ -29,6 +30,7 @@ #include #include #include +#include namespace DB::DM { @@ -104,6 +106,46 @@ ALWAYS_INLINE bool minIsNull(const DB::ColumnUInt8 & null_map, size_t i) } } // namespace details +void MinMaxIndex::appendPack( + UInt8 pack_mark, + bool has_value, + UInt8 allowed_mask, + const IColumn * column, + size_t min_idx, + size_t max_idx) +{ + RUNTIME_CHECK_MSG( + (pack_mark & ~allowed_mask) == 0, + "invalid pack_mark={:#x} allowed_mask={:#x}", + static_cast(pack_mark), + static_cast(allowed_mask)); + RUNTIME_CHECK(!has_value || column != nullptr); + + pack_marks.push_back(pack_mark); + has_value_marks.push_back(has_value ? 1 : 0); + if (has_value) + { + minmaxes->insertFrom(*column, min_idx); + minmaxes->insertFrom(*column, max_idx); + } + else + { + minmaxes->insertDefault(); + minmaxes->insertDefault(); + } +} + +bool MinMaxIndex::hasAnyTrimmedValue() const +{ + constexpr UInt8 trimmed_mask = PackMarkBits::TrimmedLow | PackMarkBits::TrimmedHigh; + for (unsigned char pack_mark : pack_marks) + { + if ((pack_mark & trimmed_mask) != 0) + return true; + } + return false; +} + void MinMaxIndex::addPack(const IColumn & column, const ColumnVector * del_mark) { auto size = column.size(); @@ -140,27 +182,18 @@ void MinMaxIndex::addPack(const IColumn & column, const ColumnVector * de std::tie(min_index, max_index) = details::minmax(column, del_mark, 0, column.size()); } + const UInt8 pack_mark = has_null ? PackMarkBits::Null : 0; if (min_index != NONE_EXIST) - { - has_null_marks.push_back(has_null); - has_value_marks.push_back(1); - minmaxes->insertFrom(column, min_index); - minmaxes->insertFrom(column, max_index); - } + appendPack(pack_mark, /*has_value*/ true, PackMarkBits::OrdinaryAllowedMask, &column, min_index, max_index); else - { - has_null_marks.push_back(has_null); - has_value_marks.push_back(0); - minmaxes->insertDefault(); - minmaxes->insertDefault(); - } + appendPack(pack_mark, /*has_value*/ false, PackMarkBits::OrdinaryAllowedMask); } void MinMaxIndex::write(const IDataType & type, WriteBuffer & buf) { - UInt64 size = has_null_marks.size(); + UInt64 size = pack_marks.size(); DB::writeIntBinary(size, buf); - buf.write(reinterpret_cast(has_null_marks.data()), sizeof(UInt8) * size); + buf.write(reinterpret_cast(pack_marks.data()), sizeof(UInt8) * size); buf.write(reinterpret_cast(has_value_marks.data()), sizeof(UInt8) * size); type.serializeBinaryBulkWithMultipleStreams( *minmaxes, // @@ -179,10 +212,10 @@ MinMaxIndexPtr MinMaxIndex::read(const IDataType & type, ReadBuffer & buf, size_ { DB::readIntBinary(size, buf); } - PaddedPODArray has_null_marks(size); + PaddedPODArray pack_marks(size); PaddedPODArray has_value_marks(size); auto minmaxes = type.createColumn(); - buf.read(reinterpret_cast(has_null_marks.data()), sizeof(UInt8) * size); + buf.read(reinterpret_cast(pack_marks.data()), sizeof(UInt8) * size); buf.read(reinterpret_cast(has_value_marks.data()), sizeof(UInt8) * size); type.deserializeBinaryBulkWithMultipleStreams( *minmaxes, // @@ -200,7 +233,22 @@ MinMaxIndexPtr MinMaxIndex::read(const IDataType & type, ReadBuffer & buf, size_ bytes_limit, bytes_read); } - return std::make_shared(std::move(has_null_marks), std::move(has_value_marks), std::move(minmaxes)); + return std::make_shared(std::move(pack_marks), std::move(has_value_marks), std::move(minmaxes)); +} + +bool MinMaxIndex::hasNull(size_t pack_index) const +{ + return hasNullMark(pack_marks[pack_index]); +} + +bool MinMaxIndex::hasTrimmedLow(size_t pack_index) const +{ + return hasTrimmedLowMark(pack_marks[pack_index]); +} + +bool MinMaxIndex::hasTrimmedHigh(size_t pack_index) const +{ + return hasTrimmedHighMark(pack_marks[pack_index]); } std::pair MinMaxIndex::getIntMinMax(size_t pack_index) const @@ -513,7 +561,7 @@ RSResults MinMaxIndex::checkNullableNullEqualImpl( /// `NULL <=> value` is false for every row, so the pack can be skipped. if (details::minIsNull(null_map, i)) { - if (has_null_marks[i] && !has_value_marks[i]) + if (hasNull(i) && !has_value_marks[i]) results[i - start_pack] = RSResult::None; continue; } @@ -523,7 +571,7 @@ RSResults MinMaxIndex::checkNullableNullEqualImpl( auto value_result = RoughCheck::CheckEqual::check(value, type, min, max); /// Non-null values may all equal `value`, but NULL rows still make `col <=> value` false. /// Downgrade All => Some so the pack is read and filtered row by row. - if (has_null_marks[i] && value_result == RSResult::All) + if (hasNull(i) && value_result == RSResult::All) results[i - start_pack] = RSResult::Some; else results[i - start_pack] = value_result; @@ -568,7 +616,7 @@ RSResults MinMaxIndex::checkNullableNullEqual( { if (details::minIsNull(null_map, i)) { - if (has_null_marks[i] && !has_value_marks[i]) + if (hasNull(i) && !has_value_marks[i]) results[i - start_pack] = RSResult::None; continue; } @@ -581,7 +629,7 @@ RSResults MinMaxIndex::checkNullableNullEqual( prev_offset = offsets[pos - 1]; auto max = String(reinterpret_cast(&chars[prev_offset]), offsets[pos] - prev_offset - 1); auto value_result = RoughCheck::CheckEqual::check(value, type, min, max); - if (has_null_marks[i] && value_result == RSResult::All) + if (hasNull(i) && value_result == RSResult::All) results[i - start_pack] = RSResult::Some; else results[i - start_pack] = value_result; @@ -732,7 +780,7 @@ RSResults MinMaxIndex::checkIsNull(size_t start_pack, size_t pack_count) RSResults results(pack_count, RSResult::None); for (size_t i = start_pack; i < start_pack + pack_count; ++i) { - if (has_null_marks[i]) + if (hasNull(i)) { results[i - start_pack] = has_value_marks[i] ? RSResult::Some : RSResult::All; } @@ -742,7 +790,7 @@ RSResults MinMaxIndex::checkIsNull(size_t start_pack, size_t pack_count) RSResult MinMaxIndex::addNullIfHasNull(RSResult value_result, size_t i) const { - if (has_null_marks[i]) + if (hasNull(i)) value_result.setHasNull(); return value_result; } @@ -750,7 +798,7 @@ RSResult MinMaxIndex::addNullIfHasNull(RSResult value_result, size_t i) const MinMaxIndex::Cell MinMaxIndex::getCell(size_t pack_index) const { Cell cell; - if (has_null_marks[pack_index]) + if (hasNull(pack_index)) cell.has_null = true; if (has_value_marks[pack_index]) { diff --git a/dbms/src/Storages/DeltaMerge/Index/MinMaxIndex.h b/dbms/src/Storages/DeltaMerge/Index/MinMaxIndex.h index 6b32501cca3..eb63b099d61 100644 --- a/dbms/src/Storages/DeltaMerge/Index/MinMaxIndex.h +++ b/dbms/src/Storages/DeltaMerge/Index/MinMaxIndex.h @@ -32,10 +32,10 @@ class MinMaxIndex { public: MinMaxIndex( - PaddedPODArray && has_null_marks_, + PaddedPODArray && pack_marks_, PaddedPODArray && has_value_marks_, MutableColumnPtr && minmaxes_) - : has_null_marks(std::move(has_null_marks_)) + : pack_marks(std::move(pack_marks_)) , has_value_marks(std::move(has_value_marks_)) , minmaxes(std::move(minmaxes_)) {} @@ -47,14 +47,27 @@ class MinMaxIndex size_t byteSize() const { // we add 3 * sizeof(PaddedPODArray) - // because has_null_marks/ has_value_marks / minmaxes are all use PaddedPODArray - // Thus we need to add the structual memory cost of PaddedPODArray for each of them - return sizeof(UInt8) * has_null_marks.size() + sizeof(UInt8) * has_value_marks.size() + minmaxes->byteSize() + // because pack_marks / has_value_marks / minmaxes all use PaddedPODArray + // Thus we need to add the structural memory cost of PaddedPODArray for each of them + return sizeof(UInt8) * pack_marks.size() + sizeof(UInt8) * has_value_marks.size() + minmaxes->byteSize() + 3 * sizeof(PaddedPODArray); } void addPack(const IColumn & column, const ColumnVector * del_mark); + /// Append one pack cell. `allowed_mask` restricts which pack-mark bits may be set + /// (ordinary: bit0 only; trim: bits 0..2). + void appendPack( + UInt8 pack_mark, + bool has_value, + UInt8 allowed_mask, + const IColumn * column = nullptr, + size_t min_idx = 0, + size_t max_idx = 0); + + /// True if any pack has trimmed-low or trimmed-high bits set. + bool hasAnyTrimmedValue() const; + void write(const IDataType & type, WriteBuffer & buf); static MinMaxIndexPtr read(const IDataType & type, ReadBuffer & buf, size_t bytes_limit); @@ -81,6 +94,17 @@ class MinMaxIndex RSResults checkIsNull(size_t start_pack, size_t pack_count); size_t size() const { return has_value_marks.size(); } + + /// Whether this pack has at least one non-NULL, non-deleted value in the index domain. + bool hasValue(size_t pack_index) const { return has_value_marks[pack_index] != 0; } + + /// Raw per-pack mark byte. Prefer hasNull() / hasTrimmedLow() / hasTrimmedHigh(). + UInt8 packMark(size_t pack_index) const { return pack_marks[pack_index]; } + bool hasNull(size_t pack_index) const; + bool hasTrimmedLow(size_t pack_index) const; + bool hasTrimmedHigh(size_t pack_index) const; + const PaddedPODArray & packMarks() const { return pack_marks; } + struct Cell { Field min; @@ -142,7 +166,9 @@ class MinMaxIndex RSResult addNullIfHasNull(RSResult value_result, size_t i) const; - PaddedPODArray has_null_marks; + // Historically named has_null_marks. Disk layout is unchanged: one UInt8 per pack. + // Bit 0 = has_null; trim indexes may also set bits 1/2. Always read NULL via hasNull(). + PaddedPODArray pack_marks; PaddedPODArray has_value_marks; MutableColumnPtr minmaxes; }; diff --git a/dbms/src/Storages/DeltaMerge/Index/RSIndex.h b/dbms/src/Storages/DeltaMerge/Index/RSIndex.h index c10b7133eba..a25e4168d42 100644 --- a/dbms/src/Storages/DeltaMerge/Index/RSIndex.h +++ b/dbms/src/Storages/DeltaMerge/Index/RSIndex.h @@ -15,6 +15,7 @@ #pragma once #include +#include namespace DB::DM { @@ -29,6 +30,14 @@ struct RSIndex {} }; +struct TrimRSIndex +{ + DataTypePtr type; + MinMaxIndexPtr minmax; + TrimMinMaxIndexMeta meta; +}; + using ColumnIndexes = std::unordered_map; +using TrimColumnIndexes = std::unordered_map; } // namespace DB::DM diff --git a/dbms/src/Storages/DeltaMerge/Index/TrimMinMaxIndex.cpp b/dbms/src/Storages/DeltaMerge/Index/TrimMinMaxIndex.cpp new file mode 100644 index 00000000000..1bf5fb85c55 --- /dev/null +++ b/dbms/src/Storages/DeltaMerge/Index/TrimMinMaxIndex.cpp @@ -0,0 +1,294 @@ +// Copyright 2026 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace DB::DM +{ +namespace +{ +const IDataType * stripNullable(const IDataType & type) +{ + if (type.isNullable()) + return static_cast(type).getNestedType().get(); + return &type; +} + +bool isSupportedTemporalNestedType(const IDataType & nested_type) +{ + return typeid_cast(&nested_type) || typeid_cast(&nested_type); +} + +const PaddedPODArray & getUInt64Data(const IColumn & column) +{ + if (column.isColumnNullable()) + { + const auto & nullable = static_cast(column); + return static_cast &>(nullable.getNestedColumn()).getData(); + } + return static_cast &>(column).getData(); +} +} // namespace + +namespace TrimMinMax +{ +bool isSupportedTemporalType(const IDataType & type) +{ + return isSupportedTemporalNestedType(*stripNullable(type)); +} + +UInt64 defaultLowerBoundPacked(const IDataType & nested_type) +{ + const auto & type = *stripNullable(nested_type); + if (typeid_cast(&type)) + return MyDate(1900, 1, 1).toPackedUInt(); + // DATETIME / TIMESTAMP share MyDateTime packing. + return MyDateTime(1900, 1, 1, 0, 0, 0, 0).toPackedUInt(); +} + +UInt64 defaultUpperBoundPacked(const IDataType & nested_type) +{ + const auto & type = *stripNullable(nested_type); + // Use 2099-12-01 rather than 2100-01-01 so TIMESTAMP / TZ / DST conversions + // near the common 2100-01-01 sentinel do not push the exclusive upper edge + // across timezone boundaries into an unexpected calendar day. + if (typeid_cast(&type)) + return MyDate(2099, 12, 1).toPackedUInt(); + return MyDateTime(2099, 12, 1, 0, 0, 0, 0).toPackedUInt(); +} + +String encodeBound(UInt64 packed) +{ + WriteBufferFromOwnString buf; + writeIntBinary(packed, buf); + return buf.str(); +} + +std::optional decodeBound(std::string_view bytes) +{ + if (bytes.size() != sizeof(UInt64)) + return std::nullopt; + ReadBufferFromMemory buf(bytes.data(), bytes.size()); + UInt64 value = 0; + readIntBinary(value, buf); + if (!buf.eof()) + return std::nullopt; + return value; +} + +dtpb::TrimMinMaxIndexProps makeDefaultProps(const IDataType & nested_type, UInt64 pack_count) +{ + dtpb::TrimMinMaxIndexProps props; + props.set_format_version(FormatVersionV1); + props.set_lower_bound(encodeBound(defaultLowerBoundPacked(nested_type))); + props.set_upper_bound(encodeBound(defaultUpperBoundPacked(nested_type))); + props.set_pack_count(pack_count); + return props; +} + +void addOrdinaryAndTrimPack( + MinMaxIndex & ordinary, + MinMaxIndex & trim, + const IColumn & column, + const ColumnVector * del_mark, + UInt64 lower_bound, + UInt64 upper_bound) +{ + const auto * del_mark_data = del_mark ? &del_mark->getData() : nullptr; + const UInt8 * null_mark_data = nullptr; + if (column.isColumnNullable()) + null_mark_data = static_cast(column).getNullMapData().data(); + + const auto & values = getUInt64Data(column); + const size_t size = column.size(); + + size_t ordinary_min_idx = size; + size_t ordinary_max_idx = size; + size_t trim_min_idx = size; + size_t trim_max_idx = size; + UInt8 trim_pack_mark = 0; + bool ordinary_has_null = false; + + for (size_t i = 0; i < size; ++i) + { + if (del_mark_data && (*del_mark_data)[i]) + continue; + + if (null_mark_data && null_mark_data[i]) + { + ordinary_has_null = true; + trim_pack_mark |= PackMarkBits::Null; + continue; + } + + const UInt64 value = values[i]; + if (ordinary_min_idx == size || value < values[ordinary_min_idx]) + ordinary_min_idx = i; + if (ordinary_max_idx == size || value > values[ordinary_max_idx]) + ordinary_max_idx = i; + + if (value >= lower_bound && value < upper_bound) + { + if (trim_min_idx == size || value < values[trim_min_idx]) + trim_min_idx = i; + if (trim_max_idx == size || value > values[trim_max_idx]) + trim_max_idx = i; + } + else if (value < lower_bound) + { + trim_pack_mark |= PackMarkBits::TrimmedLow; + } + else + { + trim_pack_mark |= PackMarkBits::TrimmedHigh; + } + } + + const UInt8 ordinary_pack_mark = ordinary_has_null ? PackMarkBits::Null : 0; + if (ordinary_min_idx != size) + { + ordinary.appendPack( + ordinary_pack_mark, + /*has_value*/ true, + PackMarkBits::OrdinaryAllowedMask, + &column, + ordinary_min_idx, + ordinary_max_idx); + } + else + { + ordinary.appendPack(ordinary_pack_mark, /*has_value*/ false, PackMarkBits::OrdinaryAllowedMask); + } + + if (trim_min_idx != size) + { + trim.appendPack( + trim_pack_mark, + /*has_value*/ true, + PackMarkBits::TrimAllowedMask, + &column, + trim_min_idx, + trim_max_idx); + } + else + { + trim.appendPack(trim_pack_mark, /*has_value*/ false, PackMarkBits::TrimAllowedMask); + } +} + +TrimMinMaxFallbackReason validateProps( + const dtpb::TrimMinMaxIndexProps & props, + const IDataType & nested_type, + UInt64 expected_pack_count, + TrimMinMaxIndexMeta * out_meta) +{ + const auto & type = *stripNullable(nested_type); + if (!isSupportedTemporalNestedType(type)) + return TrimMinMaxFallbackReason::MetadataMismatch; + + if (!props.has_format_version() || props.format_version() != FormatVersionV1) + return TrimMinMaxFallbackReason::UnsupportedVersion; + + if (!props.has_lower_bound() || !props.has_upper_bound() || !props.has_pack_count()) + return TrimMinMaxFallbackReason::MetadataMismatch; + + auto lower = decodeBound(props.lower_bound()); + auto upper = decodeBound(props.upper_bound()); + if (!lower || !upper || *lower >= *upper) + return TrimMinMaxFallbackReason::MetadataMismatch; + + if (props.pack_count() != expected_pack_count) + return TrimMinMaxFallbackReason::MetadataMismatch; + + if (out_meta) + { + out_meta->format_version = props.format_version(); + out_meta->lower_bound = *lower; + out_meta->upper_bound = *upper; + out_meta->pack_count = props.pack_count(); + } + return TrimMinMaxFallbackReason::None; +} + +TrimMinMaxFallbackReason trySelectTrimMeta( + bool read_enabled, + const std::optional & props, + const IDataType & nested_type, + UInt64 expected_pack_count, + const std::unordered_map & merged_sub_file_infos, + const String & trim_index_fname, + TrimMinMaxIndexMeta * out_meta) +{ + if (!read_enabled) + return TrimMinMaxFallbackReason::Disabled; + + if (!props.has_value()) + return TrimMinMaxFallbackReason::NoMeta; + + TrimMinMaxIndexMeta meta; + auto reason = validateProps(*props, nested_type, expected_pack_count, &meta); + if (reason != TrimMinMaxFallbackReason::None) + return reason; + + auto itr = merged_sub_file_infos.find(trim_index_fname); + if (itr == merged_sub_file_infos.end()) + return TrimMinMaxFallbackReason::IndexMissing; + + const auto & info = itr->second; + if (info.size == 0) + return TrimMinMaxFallbackReason::IndexMissing; + + meta.file_size = info.size; + meta.merged_file_number = info.number; + meta.merged_file_offset = info.offset; + if (out_meta) + *out_meta = meta; + return TrimMinMaxFallbackReason::None; +} + +bool hasOrphanTrimSubFile( + const std::optional & props, + const std::unordered_map & merged_sub_file_infos, + const String & trim_index_fname) +{ + if (props.has_value()) + return false; + auto itr = merged_sub_file_infos.find(trim_index_fname); + return itr != merged_sub_file_infos.end() && itr->second.size > 0; +} + +bool validateTrimPackMarks(const PaddedPODArray & pack_marks, size_t expected_pack_count) +{ + if (pack_marks.size() != expected_pack_count) + return false; + for (unsigned char pack_mark : pack_marks) + { + if ((pack_mark & PackMarkBits::ReservedMask) != 0) + return false; + } + return true; +} + +} // namespace TrimMinMax +} // namespace DB::DM diff --git a/dbms/src/Storages/DeltaMerge/Index/TrimMinMaxIndex.h b/dbms/src/Storages/DeltaMerge/Index/TrimMinMaxIndex.h new file mode 100644 index 00000000000..74d684fa8c4 --- /dev/null +++ b/dbms/src/Storages/DeltaMerge/Index/TrimMinMaxIndex.h @@ -0,0 +1,178 @@ +// Copyright 2026 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +namespace DB::DM +{ + +/// Pack-mark bit layout shared by ordinary and trim min-max payloads. +/// Ordinary `.idx` only uses bit 0; trim `.trim.idx` may also use bits 1/2. +namespace PackMarkBits +{ +inline constexpr UInt8 Null = 0x01; +inline constexpr UInt8 TrimmedLow = 0x02; +inline constexpr UInt8 TrimmedHigh = 0x04; +inline constexpr UInt8 ReservedMask = 0xf8; +inline constexpr UInt8 OrdinaryAllowedMask = Null; +inline constexpr UInt8 TrimAllowedMask = Null | TrimmedLow | TrimmedHigh; +} // namespace PackMarkBits + +inline bool hasNullMark(UInt8 pack_mark) +{ + return (pack_mark & PackMarkBits::Null) != 0; +} + +inline bool hasTrimmedLowMark(UInt8 pack_mark) +{ + return (pack_mark & PackMarkBits::TrimmedLow) != 0; +} + +inline bool hasTrimmedHighMark(UInt8 pack_mark) +{ + return (pack_mark & PackMarkBits::TrimmedHigh) != 0; +} + +/// Compositional wrapper around ordinary MinMaxIndex payload for trim indexes. +/// Phase A only defines the data model and validation helpers; rough-check +/// correction for low/high flags lands in Phase C. +struct TrimMinMaxIndex +{ + MinMaxIndexPtr minmax; +}; + +using TrimMinMaxIndexPtr = std::shared_ptr; + +enum class TrimMinMaxFallbackReason : UInt8 +{ + None = 0, + Disabled, + NoMeta, + UnsupportedVersion, + MetadataMismatch, + IndexMissing, + UnsupportedExpression, + PredicateBoundaryOutsideRange, + NonMetaV2, + ColumnMissing, + InvalidPackMarks, +}; + +inline std::string_view trimMinMaxFallbackReasonToString(TrimMinMaxFallbackReason reason) +{ + switch (reason) + { + case TrimMinMaxFallbackReason::None: + return "none"; + case TrimMinMaxFallbackReason::Disabled: + return "disabled"; + case TrimMinMaxFallbackReason::NoMeta: + return "no_meta"; + case TrimMinMaxFallbackReason::UnsupportedVersion: + return "unsupported_version"; + case TrimMinMaxFallbackReason::MetadataMismatch: + return "metadata_mismatch"; + case TrimMinMaxFallbackReason::IndexMissing: + return "index_missing"; + case TrimMinMaxFallbackReason::UnsupportedExpression: + return "unsupported_expression"; + case TrimMinMaxFallbackReason::PredicateBoundaryOutsideRange: + return "predicate_boundary_outside_range"; + case TrimMinMaxFallbackReason::NonMetaV2: + return "non_meta_v2"; + case TrimMinMaxFallbackReason::ColumnMissing: + return "column_missing"; + case TrimMinMaxFallbackReason::InvalidPackMarks: + return "invalid_pack_marks"; + } + return "unknown"; +} + +struct TrimMinMaxIndexMeta +{ + UInt32 format_version = 0; + UInt64 lower_bound = 0; + UInt64 upper_bound = 0; + UInt64 pack_count = 0; + UInt64 file_size = 0; + UInt64 merged_file_number = 0; + UInt64 merged_file_offset = 0; +}; + +namespace TrimMinMax +{ +inline constexpr UInt32 FormatVersionV1 = 1; + +bool isSupportedTemporalType(const IDataType & type); + +/// Default half-open effective range [1900-01-01 00:00:00, 2099-12-01 00:00:00). +UInt64 defaultLowerBoundPacked(const IDataType & nested_type); +UInt64 defaultUpperBoundPacked(const IDataType & nested_type); + +String encodeBound(UInt64 packed); +std::optional decodeBound(std::string_view bytes); + +dtpb::TrimMinMaxIndexProps makeDefaultProps(const IDataType & nested_type, UInt64 pack_count); + +/// Single-pass update of ordinary + trim min-max for one pack of MyDate/MyDateTime values. +void addOrdinaryAndTrimPack( + MinMaxIndex & ordinary, + MinMaxIndex & trim, + const IColumn & column, + const ColumnVector * del_mark, + UInt64 lower_bound, + UInt64 upper_bound); + +/// Structural validation of protobuf props against the column nested type and DMFile pack count. +/// Does not look up MergedSubFileInfo. +TrimMinMaxFallbackReason validateProps( + const dtpb::TrimMinMaxIndexProps & props, + const IDataType & nested_type, + UInt64 expected_pack_count, + TrimMinMaxIndexMeta * out_meta = nullptr); + +/// Full metadata + subfile location check used by Reader selection. +/// On success fills `out_meta` (including MergedSubFileInfo size/location). +TrimMinMaxFallbackReason trySelectTrimMeta( + bool read_enabled, + const std::optional & props, + const IDataType & nested_type, + UInt64 expected_pack_count, + const std::unordered_map & merged_sub_file_infos, + const String & trim_index_fname, + TrimMinMaxIndexMeta * out_meta = nullptr); + +/// Returns true when a `.trim.idx` MergedSubFileInfo exists but ColumnStat has no trim meta. +bool hasOrphanTrimSubFile( + const std::optional & props, + const std::unordered_map & merged_sub_file_infos, + const String & trim_index_fname); + +/// Reject reserved bits in trim pack marks. Ordinary historical marks are only 0x00/0x01. +bool validateTrimPackMarks(const PaddedPODArray & pack_marks, size_t expected_pack_count); + +} // namespace TrimMinMax + +} // namespace DB::DM diff --git a/dbms/src/Storages/DeltaMerge/Index/tests/gtest_dm_trim_minmax_index.cpp b/dbms/src/Storages/DeltaMerge/Index/tests/gtest_dm_trim_minmax_index.cpp new file mode 100644 index 00000000000..85b2a0f5df3 --- /dev/null +++ b/dbms/src/Storages/DeltaMerge/Index/tests/gtest_dm_trim_minmax_index.cpp @@ -0,0 +1,1462 @@ +// Copyright 2026 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +namespace DB::DM::tests +{ +namespace +{ +dtpb::TrimMinMaxIndexProps makeProps(UInt32 version, UInt64 lower, UInt64 upper, UInt64 pack_count) +{ + dtpb::TrimMinMaxIndexProps props; + props.set_format_version(version); + props.set_lower_bound(TrimMinMax::encodeBound(lower)); + props.set_upper_bound(TrimMinMax::encodeBound(upper)); + props.set_pack_count(pack_count); + return props; +} +} // namespace + +TEST(TrimMinMaxIndexPhaseA, PackMarkAccessors) +{ + PaddedPODArray pack_marks; + PaddedPODArray has_value_marks; + auto type = std::make_shared(0); + auto minmaxes = type->createColumn(); + + const UInt8 marks[] = {0x00, 0x01, 0x02, 0x04, 0x06, 0x07}; + for (UInt8 mark : marks) + { + pack_marks.push_back(mark); + has_value_marks.push_back(1); + minmaxes->insert(Field(static_cast(1))); + minmaxes->insert(Field(static_cast(2))); + } + + auto index = std::make_shared(std::move(pack_marks), std::move(has_value_marks), std::move(minmaxes)); + + EXPECT_FALSE(index->hasNull(0)); + EXPECT_FALSE(index->hasTrimmedLow(0)); + EXPECT_FALSE(index->hasTrimmedHigh(0)); + + EXPECT_TRUE(index->hasNull(1)); + EXPECT_FALSE(index->hasTrimmedLow(1)); + EXPECT_FALSE(index->hasTrimmedHigh(1)); + + EXPECT_FALSE(index->hasNull(2)); + EXPECT_TRUE(index->hasTrimmedLow(2)); + EXPECT_FALSE(index->hasTrimmedHigh(2)); + + EXPECT_FALSE(index->hasNull(3)); + EXPECT_FALSE(index->hasTrimmedLow(3)); + EXPECT_TRUE(index->hasTrimmedHigh(3)); + + EXPECT_FALSE(index->hasNull(4)); + EXPECT_TRUE(index->hasTrimmedLow(4)); + EXPECT_TRUE(index->hasTrimmedHigh(4)); + + EXPECT_TRUE(index->hasNull(5)); + EXPECT_TRUE(index->hasTrimmedLow(5)); + EXPECT_TRUE(index->hasTrimmedHigh(5)); + + // Ordinary NULL checks must ignore trim bits (bit1/bit2 must not look like NULL). + EXPECT_FALSE(hasNullMark(0x02)); + EXPECT_FALSE(hasNullMark(0x04)); + EXPECT_FALSE(hasNullMark(0x06)); + EXPECT_TRUE(hasNullMark(0x07)); +} + +TEST(TrimMinMaxIndexPhaseA, ValidateTrimPackMarks) +{ + PaddedPODArray ok; + ok.push_back(0x00); + ok.push_back(0x01); + ok.push_back(0x02); + ok.push_back(0x04); + ok.push_back(0x07); + EXPECT_TRUE(TrimMinMax::validateTrimPackMarks(ok, 5)); + + PaddedPODArray bad_reserved; + bad_reserved.push_back(0x08); + EXPECT_FALSE(TrimMinMax::validateTrimPackMarks(bad_reserved, 1)); + + PaddedPODArray bad_count; + bad_count.push_back(0x01); + EXPECT_FALSE(TrimMinMax::validateTrimPackMarks(bad_count, 2)); +} + +TEST(TrimMinMaxIndexPhaseA, ColumnStatProtoRoundTripAndUnknownField) +{ + auto type = std::make_shared(0); + ColumnStat stat{ + .col_id = 42, + .type = type, + .avg_size = 8, + .serialized_bytes = 100, + .data_bytes = 80, + .mark_bytes = 8, + .index_bytes = 20, + .indexes = {}, + .trim_minmax_index = TrimMinMax::makeDefaultProps(*type, /*pack_count*/ 3), + }; + + auto proto = stat.toProto(); + ASSERT_TRUE(proto.has_trim_minmax_index()); + EXPECT_EQ(proto.trim_minmax_index().format_version(), TrimMinMax::FormatVersionV1); + EXPECT_EQ(proto.trim_minmax_index().pack_count(), 3u); + // Must not land in indexes=104. + EXPECT_EQ(proto.indexes_size(), 0); + + ColumnStat restored; + restored.mergeFromProto(proto); + ASSERT_TRUE(restored.trim_minmax_index.has_value()); + EXPECT_EQ(restored.trim_minmax_index->pack_count(), 3u); + EXPECT_EQ(restored.col_id, 42); + + // Simulate an old Reader that only knows fields up to indexes=104: + // serialize with field 105, then parse into a message and clear unknown fields + // by copying only known fields used by old code paths. + String bytes; + ASSERT_TRUE(proto.SerializeToString(&bytes)); + + dtpb::ColumnStat old_reader_view; + ASSERT_TRUE(old_reader_view.ParseFromString(bytes)); + // Current generated code knows field 105; verify wire bytes still round-trip when + // field 105 is dropped (metadata rewrite by an old node). + old_reader_view.clear_trim_minmax_index(); + ColumnStat after_old_rewrite; + after_old_rewrite.mergeFromProto(old_reader_view); + EXPECT_FALSE(after_old_rewrite.trim_minmax_index.has_value()); +} + +TEST(TrimMinMaxIndexPhaseA, TrySelectTrimMetaFallbackReasons) +{ + auto type = std::make_shared(0); + const auto lower = TrimMinMax::defaultLowerBoundPacked(*type); + const auto upper = TrimMinMax::defaultUpperBoundPacked(*type); + const String fname = colTrimIndexFileName("42"); + + std::unordered_map merged; + merged.emplace(fname, MergedSubFileInfo(fname, /*number*/ 0, /*offset*/ 10, /*size*/ 100)); + + TrimMinMaxIndexMeta meta; + EXPECT_EQ( + TrimMinMax::trySelectTrimMeta( + /*read_enabled*/ false, + makeProps(1, lower, upper, 2), + *type, + /*expected_pack_count*/ 2, + merged, + fname, + &meta), + TrimMinMaxFallbackReason::Disabled); + + EXPECT_EQ( + TrimMinMax::trySelectTrimMeta( + /*read_enabled*/ true, + std::nullopt, + *type, + 2, + merged, + fname, + &meta), + TrimMinMaxFallbackReason::NoMeta); + + EXPECT_EQ( + TrimMinMax::trySelectTrimMeta( + /*read_enabled*/ true, + makeProps(/*version*/ 99, lower, upper, 2), + *type, + 2, + merged, + fname, + &meta), + TrimMinMaxFallbackReason::UnsupportedVersion); + + EXPECT_EQ( + TrimMinMax::trySelectTrimMeta( + /*read_enabled*/ true, + makeProps(1, upper, lower, 2), // lower >= upper + *type, + 2, + merged, + fname, + &meta), + TrimMinMaxFallbackReason::MetadataMismatch); + + EXPECT_EQ( + TrimMinMax::trySelectTrimMeta( + /*read_enabled*/ true, + makeProps(1, lower, upper, /*pack_count*/ 3), + *type, + /*expected*/ 2, + merged, + fname, + &meta), + TrimMinMaxFallbackReason::MetadataMismatch); + + auto props_ok = makeProps(1, lower, upper, 2); + EXPECT_EQ( + TrimMinMax::trySelectTrimMeta( + /*read_enabled*/ true, + props_ok, + *type, + 2, + /*empty map*/ {}, + fname, + &meta), + TrimMinMaxFallbackReason::IndexMissing); + + EXPECT_EQ( + TrimMinMax::trySelectTrimMeta(/*read_enabled*/ true, props_ok, *type, 2, merged, fname, &meta), + TrimMinMaxFallbackReason::None); + EXPECT_EQ(meta.pack_count, 2u); + EXPECT_EQ(meta.file_size, 100u); + EXPECT_EQ(meta.lower_bound, lower); + EXPECT_EQ(meta.upper_bound, upper); + + // Orphan subfile: meta missing but MergedSubFileInfo remains. + EXPECT_TRUE(TrimMinMax::hasOrphanTrimSubFile(std::nullopt, merged, fname)); + EXPECT_FALSE(TrimMinMax::hasOrphanTrimSubFile(props_ok, merged, fname)); +} + +TEST(TrimMinMaxIndexPhaseA, FileNamingAndCacheKeyDistinct) +{ + EXPECT_EQ(colIndexFileName("42"), "42.idx"); + EXPECT_EQ(colTrimIndexFileName("42"), "42.trim.idx"); + EXPECT_NE(colIndexFileName("42"), colTrimIndexFileName("42")); + EXPECT_TRUE(endsWith(colTrimIndexFileName("42"), details::TRIM_INDEX_FILE_SUFFIX)); + EXPECT_TRUE(endsWith(colTrimIndexFileName("42"), details::INDEX_FILE_SUFFIX)); +} + +TEST(TrimMinMaxIndexPhaseA, SettingsDefaultOff) +{ + Settings settings; + EXPECT_FALSE(settings.dt_enable_trim_minmax); +} + +TEST(TrimMinMaxIndexPhaseB, AddOrdinaryAndTrimPack) +{ + auto type = std::make_shared(0); + MinMaxIndex ordinary(*type); + MinMaxIndex trim(*type); + + const UInt64 lower = TrimMinMax::defaultLowerBoundPacked(*type); + const UInt64 upper = TrimMinMax::defaultUpperBoundPacked(*type); + const UInt64 in_range = MyDateTime(2020, 1, 1, 0, 0, 0, 0).toPackedUInt(); + const UInt64 low_out = MyDateTime(1800, 1, 1, 0, 0, 0, 0).toPackedUInt(); + const UInt64 high_out = MyDateTime(2100, 1, 1, 0, 0, 0, 0).toPackedUInt(); + + auto make_col = [&](const std::vector & vals) { + auto col = type->createColumn(); + for (auto v : vals) + col->insert(Field(v)); + return col; + }; + + // Pack 0: all in range + { + auto col = make_col({in_range, in_range + 1}); + TrimMinMax::addOrdinaryAndTrimPack(ordinary, trim, *col, nullptr, lower, upper); + EXPECT_FALSE(ordinary.hasNull(0)); + EXPECT_TRUE(ordinary.getCell(0).has_value); + EXPECT_FALSE(trim.hasTrimmedLow(0)); + EXPECT_FALSE(trim.hasTrimmedHigh(0)); + EXPECT_TRUE(trim.getCell(0).has_value); + } + + // Pack 1: high outlier only + { + auto col = make_col({high_out}); + TrimMinMax::addOrdinaryAndTrimPack(ordinary, trim, *col, nullptr, lower, upper); + EXPECT_TRUE(ordinary.getCell(1).has_value); + EXPECT_FALSE(trim.getCell(1).has_value); + EXPECT_FALSE(trim.hasTrimmedLow(1)); + EXPECT_TRUE(trim.hasTrimmedHigh(1)); + } + + // Pack 2: low + high + in-range + { + auto col = make_col({low_out, in_range, high_out}); + TrimMinMax::addOrdinaryAndTrimPack(ordinary, trim, *col, nullptr, lower, upper); + EXPECT_TRUE(trim.getCell(2).has_value); + EXPECT_TRUE(trim.hasTrimmedLow(2)); + EXPECT_TRUE(trim.hasTrimmedHigh(2)); + EXPECT_EQ(trim.getCell(2).min.safeGet(), in_range); + EXPECT_EQ(trim.getCell(2).max.safeGet(), in_range); + } + + // Pack 3: nullable NULL + high outlier + { + auto nullable_type = makeNullable(type); + MinMaxIndex ordinary_n(*nullable_type); + MinMaxIndex trim_n(*nullable_type); + auto col = nullable_type->createColumn(); + col->insertDefault(); // NULL + col->insert(Field(high_out)); + TrimMinMax::addOrdinaryAndTrimPack(ordinary_n, trim_n, *col, nullptr, lower, upper); + EXPECT_TRUE(ordinary_n.hasNull(0)); + EXPECT_TRUE(trim_n.hasNull(0)); + EXPECT_TRUE(trim_n.hasTrimmedHigh(0)); + EXPECT_FALSE(trim_n.getCell(0).has_value); + EXPECT_EQ(trim_n.packMark(0), PackMarkBits::Null | PackMarkBits::TrimmedHigh); + } + + EXPECT_TRUE(trim.hasAnyTrimmedValue()); + EXPECT_FALSE(ordinary.hasAnyTrimmedValue()); + + // Serialize / deserialize trim payload + WriteBufferFromOwnString buf; + trim.write(*type, buf); + ReadBufferFromString rbuf(buf.str()); + auto restored = MinMaxIndex::read(*type, rbuf, buf.str().size()); + ASSERT_TRUE(TrimMinMax::validateTrimPackMarks(restored->packMarks(), 3)); + EXPECT_TRUE(restored->hasTrimmedHigh(1)); + EXPECT_TRUE(restored->hasTrimmedLow(2)); + EXPECT_TRUE(restored->hasTrimmedHigh(2)); +} + +// Design Validation Strategy: NULL-only, delete-mark, and no-valid-value packs. +TEST(TrimMinMaxIndexPhaseB, AddOrdinaryAndTrimPackNullDeleteAndEmpty) +{ + auto type = std::make_shared(0); + auto nullable_type = makeNullable(type); + const UInt64 lower = TrimMinMax::defaultLowerBoundPacked(*type); + const UInt64 upper = TrimMinMax::defaultUpperBoundPacked(*type); + const UInt64 in_range = MyDateTime(2020, 1, 1, 0, 0, 0, 0).toPackedUInt(); + const UInt64 high_out = MyDateTime(2100, 1, 1, 0, 0, 0, 0).toPackedUInt(); + const UInt64 low_out = MyDateTime(1800, 1, 1, 0, 0, 0, 0).toPackedUInt(); + + // NULL only: has_null, no min/max, no low/high trimmed bits. + { + MinMaxIndex ordinary(*nullable_type); + MinMaxIndex trim(*nullable_type); + auto col = nullable_type->createColumn(); + col->insertDefault(); + col->insertDefault(); + TrimMinMax::addOrdinaryAndTrimPack(ordinary, trim, *col, nullptr, lower, upper); + + EXPECT_TRUE(ordinary.hasNull(0)); + EXPECT_FALSE(ordinary.hasValue(0)); + EXPECT_TRUE(trim.hasNull(0)); + EXPECT_FALSE(trim.hasValue(0)); + EXPECT_FALSE(trim.hasTrimmedLow(0)); + EXPECT_FALSE(trim.hasTrimmedHigh(0)); + EXPECT_EQ(trim.packMark(0), PackMarkBits::Null); + EXPECT_FALSE(trim.hasAnyTrimmedValue()); + } + + // Delete mark + normal + outlier: deleted rows must not participate in min/max or flags. + { + MinMaxIndex ordinary(*type); + MinMaxIndex trim(*type); + auto col = type->createColumn(); + col->insert(Field(high_out)); // deleted high outlier + col->insert(Field(in_range)); // live in-range + col->insert(Field(low_out)); // deleted low outlier + auto del_mark_col = createColumn({1, 0, 1}).column; + const auto * del_mark = static_cast *>(del_mark_col.get()); + + TrimMinMax::addOrdinaryAndTrimPack(ordinary, trim, *col, del_mark, lower, upper); + + EXPECT_TRUE(ordinary.hasValue(0)); + EXPECT_EQ(ordinary.getCell(0).min.safeGet(), in_range); + EXPECT_EQ(ordinary.getCell(0).max.safeGet(), in_range); + EXPECT_TRUE(trim.hasValue(0)); + EXPECT_EQ(trim.getCell(0).min.safeGet(), in_range); + EXPECT_EQ(trim.getCell(0).max.safeGet(), in_range); + EXPECT_FALSE(trim.hasTrimmedLow(0)); + EXPECT_FALSE(trim.hasTrimmedHigh(0)); + EXPECT_EQ(trim.packMark(0), 0); + EXPECT_FALSE(trim.hasAnyTrimmedValue()); + } + + // No valid values: empty column. + { + MinMaxIndex ordinary(*type); + MinMaxIndex trim(*type); + auto col = type->createColumn(); + TrimMinMax::addOrdinaryAndTrimPack(ordinary, trim, *col, nullptr, lower, upper); + + EXPECT_FALSE(ordinary.hasValue(0)); + EXPECT_FALSE(ordinary.hasNull(0)); + EXPECT_FALSE(trim.hasValue(0)); + EXPECT_FALSE(trim.hasNull(0)); + EXPECT_FALSE(trim.hasTrimmedLow(0)); + EXPECT_FALSE(trim.hasTrimmedHigh(0)); + EXPECT_EQ(trim.packMark(0), 0); + EXPECT_FALSE(trim.hasAnyTrimmedValue()); + } + + // No valid values: all rows deleted (including outliers that must be ignored). + { + MinMaxIndex ordinary(*type); + MinMaxIndex trim(*type); + auto col = type->createColumn(); + col->insert(Field(high_out)); + col->insert(Field(low_out)); + auto del_mark_col = createColumn({1, 1}).column; + const auto * del_mark = static_cast *>(del_mark_col.get()); + + TrimMinMax::addOrdinaryAndTrimPack(ordinary, trim, *col, del_mark, lower, upper); + + EXPECT_FALSE(ordinary.hasValue(0)); + EXPECT_FALSE(trim.hasValue(0)); + EXPECT_FALSE(trim.hasTrimmedLow(0)); + EXPECT_FALSE(trim.hasTrimmedHigh(0)); + EXPECT_EQ(trim.packMark(0), 0); + EXPECT_FALSE(trim.hasAnyTrimmedValue()); + } + + // Deleted NULL must not set has_null; live outlier still sets trim flags. + { + MinMaxIndex ordinary(*nullable_type); + MinMaxIndex trim(*nullable_type); + auto col = nullable_type->createColumn(); + col->insertDefault(); // deleted NULL + col->insert(Field(high_out)); // live high outlier + auto del_mark_col = createColumn({1, 0}).column; + const auto * del_mark = static_cast *>(del_mark_col.get()); + + TrimMinMax::addOrdinaryAndTrimPack(ordinary, trim, *col, del_mark, lower, upper); + + EXPECT_FALSE(ordinary.hasNull(0)); + EXPECT_TRUE(ordinary.hasValue(0)); + EXPECT_FALSE(trim.hasNull(0)); + EXPECT_FALSE(trim.hasValue(0)); + EXPECT_TRUE(trim.hasTrimmedHigh(0)); + EXPECT_EQ(trim.packMark(0), PackMarkBits::TrimmedHigh); + EXPECT_TRUE(trim.hasAnyTrimmedValue()); + } +} + +TEST(TrimMinMaxIndexPhaseB, AppendPackRejectsInvalidMask) +{ + auto type = std::make_shared(0); + MinMaxIndex index(*type); + EXPECT_THROW( + index.appendPack(/*pack_mark*/ 0x08, /*has_value*/ false, PackMarkBits::TrimAllowedMask), + DB::Exception); + EXPECT_THROW( + index + .appendPack(/*pack_mark*/ PackMarkBits::TrimmedLow, /*has_value*/ false, PackMarkBits::OrdinaryAllowedMask), + DB::Exception); +} + +// Design Validation Strategy β€” Temporal-Type Tests: +// DATE / DATETIME(fsp) bounds, FSP near upper edge, zero/invalid packed values, Nullable. +TEST(TrimMinMaxIndexTemporalTypes, DateAndDateTimeBoundsFspAndCompatValues) +{ + auto expect_classification + = [](const DataTypePtr & type, UInt64 value, bool expect_has_value, bool expect_low, bool expect_high) { + MinMaxIndex ordinary(*type); + MinMaxIndex trim(*type); + const UInt64 lower = TrimMinMax::defaultLowerBoundPacked(*type); + const UInt64 upper = TrimMinMax::defaultUpperBoundPacked(*type); + auto col = type->createColumn(); + col->insert(Field(value)); + TrimMinMax::addOrdinaryAndTrimPack(ordinary, trim, *col, nullptr, lower, upper); + EXPECT_EQ(trim.hasValue(0), expect_has_value) << type->getName() << " value=" << value; + EXPECT_EQ(trim.hasTrimmedLow(0), expect_low) << type->getName() << " value=" << value; + EXPECT_EQ(trim.hasTrimmedHigh(0), expect_high) << type->getName() << " value=" << value; + if (expect_has_value) + { + EXPECT_EQ(trim.getCell(0).min.safeGet(), value); + EXPECT_EQ(trim.getCell(0).max.safeGet(), value); + } + }; + + // Supported types: DATE, DATETIME(0/3/6), Nullable wrappers. + EXPECT_TRUE(TrimMinMax::isSupportedTemporalType(*std::make_shared())); + EXPECT_TRUE(TrimMinMax::isSupportedTemporalType(*std::make_shared(0))); + EXPECT_TRUE(TrimMinMax::isSupportedTemporalType(*std::make_shared(3))); + EXPECT_TRUE(TrimMinMax::isSupportedTemporalType(*std::make_shared(6))); + EXPECT_TRUE(TrimMinMax::isSupportedTemporalType(*makeNullable(std::make_shared()))); + EXPECT_TRUE(TrimMinMax::isSupportedTemporalType(*makeNullable(std::make_shared(6)))); + + // DATE and DATETIME default bounds use type-specific packing (FSPTT differs). + auto date_type = std::make_shared(); + auto dt0 = std::make_shared(0); + auto dt3 = std::make_shared(3); + auto dt6 = std::make_shared(6); + + EXPECT_EQ(TrimMinMax::defaultLowerBoundPacked(*date_type), MyDate(1900, 1, 1).toPackedUInt()); + EXPECT_EQ(TrimMinMax::defaultUpperBoundPacked(*date_type), MyDate(2099, 12, 1).toPackedUInt()); + EXPECT_EQ(TrimMinMax::defaultLowerBoundPacked(*dt0), MyDateTime(1900, 1, 1, 0, 0, 0, 0).toPackedUInt()); + EXPECT_EQ(TrimMinMax::defaultUpperBoundPacked(*dt0), MyDateTime(2099, 12, 1, 0, 0, 0, 0).toPackedUInt()); + // FSP does not change the persisted default packed bounds. + EXPECT_EQ(TrimMinMax::defaultLowerBoundPacked(*dt3), TrimMinMax::defaultLowerBoundPacked(*dt0)); + EXPECT_EQ(TrimMinMax::defaultUpperBoundPacked(*dt6), TrimMinMax::defaultUpperBoundPacked(*dt0)); + // TiFlash DATE / DATETIME share the same packed integer layout for midnight values. + EXPECT_EQ(TrimMinMax::defaultLowerBoundPacked(*date_type), TrimMinMax::defaultLowerBoundPacked(*dt0)); + EXPECT_EQ(TrimMinMax::defaultUpperBoundPacked(*date_type), TrimMinMax::defaultUpperBoundPacked(*dt0)); + EXPECT_EQ(MyDate(2020, 1, 1).toPackedUInt(), MyDateTime(2020, 1, 1, 0, 0, 0, 0).toPackedUInt()); + + // DATE: inclusive lower / exclusive upper / zero-date low outlier. + expect_classification(date_type, MyDate(1900, 1, 1).toPackedUInt(), /*has*/ true, /*low*/ false, /*high*/ false); + expect_classification(date_type, MyDate(2099, 11, 30).toPackedUInt(), true, false, false); + expect_classification(date_type, MyDate(2099, 12, 1).toPackedUInt(), false, false, true); + expect_classification(date_type, MyDate(2100, 1, 1).toPackedUInt(), false, false, true); + expect_classification(date_type, MyDate(0, 0, 0).toPackedUInt(), false, true, false); + // Invalid-date compatibility packed value still participates via packed compare (inside E). + expect_classification(date_type, MyDate(2020, 2, 30).toPackedUInt(), true, false, false); + + // DATETIME(0/3/6): half-open E covers 2099-11-30 23:59:59.999999; 2099-12-01 is high. + const UInt64 just_inside = MyDateTime(2099, 11, 30, 23, 59, 59, 999999).toPackedUInt(); + const UInt64 upper_edge = MyDateTime(2099, 12, 1, 0, 0, 0, 0).toPackedUInt(); + const UInt64 lower_edge = MyDateTime(1900, 1, 1, 0, 0, 0, 0).toPackedUInt(); + const UInt64 zero_dt = MyDateTime(0, 0, 0, 0, 0, 0, 0).toPackedUInt(); + for (const auto & type : {dt0, dt3, dt6}) + { + expect_classification(type, lower_edge, true, false, false); + expect_classification(type, just_inside, true, false, false); + expect_classification(type, upper_edge, false, false, true); + expect_classification(type, MyDateTime(2100, 1, 1, 0, 0, 0, 0).toPackedUInt(), false, false, true); + expect_classification(type, zero_dt, false, true, false); + // Fractional second inside a normal day stays in-range for every fsp. + expect_classification(type, MyDateTime(2020, 1, 1, 12, 0, 0, 123456).toPackedUInt(), true, false, false); + } + + // Nullable DATE: NULL + in-range value. + { + auto nullable_date = makeNullable(date_type); + MinMaxIndex ordinary(*nullable_date); + MinMaxIndex trim(*nullable_date); + const UInt64 lower = TrimMinMax::defaultLowerBoundPacked(*nullable_date); + const UInt64 upper = TrimMinMax::defaultUpperBoundPacked(*nullable_date); + auto col = nullable_date->createColumn(); + col->insertDefault(); + col->insert(Field(MyDate(2020, 1, 1).toPackedUInt())); + TrimMinMax::addOrdinaryAndTrimPack(ordinary, trim, *col, nullptr, lower, upper); + EXPECT_TRUE(trim.hasNull(0)); + EXPECT_TRUE(trim.hasValue(0)); + EXPECT_FALSE(trim.hasTrimmedLow(0)); + EXPECT_FALSE(trim.hasTrimmedHigh(0)); + EXPECT_EQ(trim.getCell(0).min.safeGet(), MyDate(2020, 1, 1).toPackedUInt()); + } + + // NotNull DATE pack with only in-range values must not set null/low/high. + { + MinMaxIndex ordinary(*date_type); + MinMaxIndex trim(*date_type); + const UInt64 lower = TrimMinMax::defaultLowerBoundPacked(*date_type); + const UInt64 upper = TrimMinMax::defaultUpperBoundPacked(*date_type); + auto col = date_type->createColumn(); + col->insert(Field(MyDate(2020, 6, 1).toPackedUInt())); + col->insert(Field(MyDate(2021, 6, 1).toPackedUInt())); + TrimMinMax::addOrdinaryAndTrimPack(ordinary, trim, *col, nullptr, lower, upper); + EXPECT_FALSE(ordinary.hasNull(0)); + EXPECT_FALSE(trim.hasNull(0)); + EXPECT_TRUE(trim.hasValue(0)); + EXPECT_FALSE(trim.hasTrimmedLow(0)); + EXPECT_FALSE(trim.hasTrimmedHigh(0)); + EXPECT_FALSE(trim.hasAnyTrimmedValue()); + } + + // Props encode/decode preserves DATE bounds. + { + auto props = TrimMinMax::makeDefaultProps(*date_type, /*pack_count*/ 4); + EXPECT_EQ(props.format_version(), TrimMinMax::FormatVersionV1); + EXPECT_EQ(TrimMinMax::decodeBound(props.lower_bound()), MyDate(1900, 1, 1).toPackedUInt()); + EXPECT_EQ(TrimMinMax::decodeBound(props.upper_bound()), MyDate(2099, 12, 1).toPackedUInt()); + } +} + +// Eligibility: DATE calendar values and DATETIME FSP edge values against half-open E. +TEST(TrimMinMaxIndexTemporalTypes, DateQueryDomainUsesTypePackedBounds) +{ + const UInt64 date_lo = MyDate(1900, 1, 1).toPackedUInt(); + const UInt64 date_hi = MyDate(2099, 12, 1).toPackedUInt(); + const UInt64 dt_lo = MyDateTime(1900, 1, 1, 0, 0, 0, 0).toPackedUInt(); + const UInt64 dt_hi = MyDateTime(2099, 12, 1, 0, 0, 0, 0).toPackedUInt(); + + DateQueryDomain d; + d.predicate_class = TrimPredicateClass::EqualityOrInOrBounded; + + d.values = {Field(MyDate(2020, 1, 1).toPackedUInt())}; + EXPECT_TRUE(d.isTrimEligible(date_lo, date_hi)); + d.values = {Field(MyDate(2099, 12, 1).toPackedUInt())}; + EXPECT_FALSE(d.isTrimEligible(date_lo, date_hi)); + d.values = {Field(MyDate(2100, 1, 1).toPackedUInt())}; + EXPECT_FALSE(d.isTrimEligible(date_lo, date_hi)); + d.values = {Field(MyDate(0, 0, 0).toPackedUInt())}; + EXPECT_FALSE(d.isTrimEligible(date_lo, date_hi)); + + // Half-open E keeps 2099-11-30 23:59:59.999999 eligible; 2099-12-01 / 2100-01-01 are not. + d.values = {Field(MyDateTime(2099, 11, 30, 23, 59, 59, 999999).toPackedUInt())}; + EXPECT_TRUE(d.isTrimEligible(dt_lo, dt_hi)); + d.values = {Field(MyDateTime(2099, 12, 1, 0, 0, 0, 0).toPackedUInt())}; + EXPECT_FALSE(d.isTrimEligible(dt_lo, dt_hi)); + d.values = {Field(MyDateTime(2100, 1, 1, 0, 0, 0, 0).toPackedUInt())}; + EXPECT_FALSE(d.isTrimEligible(dt_lo, dt_hi)); + + // One-sided lower bound at the FSP edge remains eligible. + d.predicate_class = TrimPredicateClass::LowerBounded; + d.values.clear(); + d.lower = Field(MyDateTime(2099, 11, 30, 23, 59, 59, 999999).toPackedUInt()); + EXPECT_TRUE(d.isTrimEligible(dt_lo, dt_hi)); + d.lower = Field(MyDateTime(2099, 12, 1, 0, 0, 0, 0).toPackedUInt()); + EXPECT_FALSE(d.isTrimEligible(dt_lo, dt_hi)); +} + +class TrimMinMaxIndexWriteTest : public DB::base::TiFlashStorageTestBasic +{ +protected: + void SetUp() override + { + TiFlashStorageTestBasic::SetUp(); + parent_path = getTemporaryPath(); + file_provider = db_context->getFileProvider(); + } + + static constexpr ColId settle_col_id = 100; + + static ColumnDefinesPtr makeColumns() + { + auto cols = DMTestEnv::getDefaultColumns(DMTestEnv::PkType::HiddenTiDBRowID, /*add_nullable*/ false); + cols->emplace_back(ColumnDefine{settle_col_id, "settle_time", std::make_shared(0)}); + return cols; + } + + static Block makeBlockWithOutlier(size_t rows) + { + Block block = DMTestEnv::prepareSimpleWriteBlock(0, rows, /*reversed*/ false); + auto type = std::make_shared(0); + auto col = type->createColumn(); + const UInt64 normal = MyDateTime(2020, 6, 1, 0, 0, 0, 0).toPackedUInt(); + const UInt64 sentinel = MyDateTime(2100, 1, 1, 0, 0, 0, 0).toPackedUInt(); + for (size_t i = 0; i < rows; ++i) + col->insert(Field(i == 0 ? sentinel : normal)); + block.insert(ColumnWithTypeAndName(std::move(col), type, "settle_time", settle_col_id)); + return block; + } + + static Block makeBlockInRangeOnly(size_t rows) + { + Block block = DMTestEnv::prepareSimpleWriteBlock(0, rows, /*reversed*/ false); + auto type = std::make_shared(0); + auto col = type->createColumn(); + const UInt64 normal = MyDateTime(2020, 6, 1, 0, 0, 0, 0).toPackedUInt(); + for (size_t i = 0; i < rows; ++i) + col->insert(Field(normal + i)); + block.insert(ColumnWithTypeAndName(std::move(col), type, "settle_time", settle_col_id)); + return block; + } + + DMFilePtr writeDMFile(const Block & block, bool enable_trim_write) + { + auto dm_file = DMFile::create( + /*file_id*/ 1, + parent_path, + std::make_optional(), + 128 * 1024, + 16 * 1024 * 1024, + DMFileFormat::V3); + auto cols = makeColumns(); + DMFileWriter::Options options; + options.enable_trim_minmax = enable_trim_write; + DMFileWriter writer(dm_file, *cols, file_provider, db_context->getWriteLimiter(), options); + writer.write(block, DMFileWriter::BlockProperty{0, 0, 0, 0}); + writer.finalize(); + return dm_file; + } + + String parent_path; + FileProviderPtr file_provider; +}; + +TEST_F(TrimMinMaxIndexWriteTest, WriteTrimIndexWhenOutliersExist) +try +{ + auto dm_file = writeDMFile(makeBlockWithOutlier(/*rows*/ 8), /*enable_trim_write*/ true); + ASSERT_TRUE(dm_file->getColumnStat(settle_col_id).trim_minmax_index.has_value()); + EXPECT_EQ(dm_file->getColumnStat(settle_col_id).trim_minmax_index->pack_count(), dm_file->getPacks()); + EXPECT_EQ(dm_file->getColumnStat(settle_col_id).trim_minmax_index->format_version(), TrimMinMax::FormatVersionV1); + + const auto * meta = typeid_cast(dm_file->meta.get()); + ASSERT_NE(meta, nullptr); + const auto fname = colTrimIndexFileName(DB::toString(settle_col_id)); + auto itr = meta->merged_sub_file_infos.find(fname); + ASSERT_NE(itr, meta->merged_sub_file_infos.end()); + EXPECT_GT(itr->second.size, 0u); + + TrimMinMaxIndexMeta selected; + auto reason = TrimMinMax::trySelectTrimMeta( + /*read_enabled*/ true, + dm_file->getColumnStat(settle_col_id).trim_minmax_index, + *dm_file->getColumnStat(settle_col_id).type, + dm_file->getPacks(), + meta->merged_sub_file_infos, + fname, + &selected); + EXPECT_EQ(reason, TrimMinMaxFallbackReason::None); + EXPECT_EQ(selected.file_size, itr->second.size); + + // Restore and verify meta survives MetaV2 round-trip. + auto restored = DMFile::restore( + file_provider, + 1, + 1, + parent_path, + DMFileMeta::ReadMode::all(), + /*meta_version*/ 0); + ASSERT_TRUE(restored->getColumnStat(settle_col_id).trim_minmax_index.has_value()); +} +CATCH + +TEST_F(TrimMinMaxIndexWriteTest, SkipPersistWhenNoOutliers) +try +{ + auto dm_file = writeDMFile(makeBlockInRangeOnly(/*rows*/ 8), /*enable_trim_write*/ true); + EXPECT_FALSE(dm_file->getColumnStat(settle_col_id).trim_minmax_index.has_value()); + + const auto * meta = typeid_cast(dm_file->meta.get()); + ASSERT_NE(meta, nullptr); + const auto fname = colTrimIndexFileName(DB::toString(settle_col_id)); + EXPECT_EQ(meta->merged_sub_file_infos.find(fname), meta->merged_sub_file_infos.end()); +} +CATCH + +TEST_F(TrimMinMaxIndexWriteTest, DisabledWriteSkipsTrim) +try +{ + auto dm_file = writeDMFile(makeBlockWithOutlier(/*rows*/ 8), /*enable_trim_write*/ false); + EXPECT_FALSE(dm_file->getColumnStat(settle_col_id).trim_minmax_index.has_value()); + + const auto * meta = typeid_cast(dm_file->meta.get()); + ASSERT_NE(meta, nullptr); + const auto fname = colTrimIndexFileName(DB::toString(settle_col_id)); + EXPECT_EQ(meta->merged_sub_file_infos.find(fname), meta->merged_sub_file_infos.end()); +} +CATCH + +TEST(TrimMinMaxIndexPhaseC, DateQueryDomainEligibility) +{ + // E = {date| date ∈ [1900-01-01 00:00:00, 2099-12-01 00:00:00)} + const UInt64 e_lo = MyDateTime(1900, 1, 1, 0, 0, 0, 0).toPackedUInt(); + const UInt64 e_hi = MyDateTime(2099, 12, 1, 0, 0, 0, 0).toPackedUInt(); + const UInt64 in_e = MyDateTime(2020, 1, 1, 0, 0, 0, 0).toPackedUInt(); + const UInt64 below = MyDateTime(1800, 1, 1, 0, 0, 0, 0).toPackedUInt(); + const UInt64 above = MyDateTime(2200, 1, 1, 0, 0, 0, 0).toPackedUInt(); + + { + // isTrimEligible == true when Q = {date | date ∈ {in_e}} (equality) + DateQueryDomain d; + d.predicate_class = TrimPredicateClass::EqualityOrInOrBounded; + d.values = {Field(in_e)}; + EXPECT_TRUE(d.isTrimEligible(e_lo, e_hi)); + d.values = {Field(above)}; + EXPECT_FALSE(d.isTrimEligible(e_lo, e_hi)); + // lower bound is inclusive, upper bound is exclusive. + d.values = {Field(e_lo)}; + EXPECT_TRUE(d.isTrimEligible(e_lo, e_hi)); + d.values = {Field(e_hi)}; + EXPECT_FALSE(d.isTrimEligible(e_lo, e_hi)); + } + { + // isTrimEligible == true when Q = {date | date ∈ [in_e, in_e + 1]} (bounded range) + DateQueryDomain d; + d.predicate_class = TrimPredicateClass::EqualityOrInOrBounded; + d.lower = Field(in_e); + d.upper = Field(in_e + 1); + EXPECT_TRUE(d.isTrimEligible(e_lo, e_hi)); + // isTrimEligible == false when Q = {date | date ∈ [ine_e, above]} (bounded range) + d.upper = Field(above); + EXPECT_FALSE(d.isTrimEligible(e_lo, e_hi)); + } + { + // isTrimEligible == true when Q = {date | date ∈ [in_e, ∞)} (lower-bounded) + DateQueryDomain d; + d.predicate_class = TrimPredicateClass::LowerBounded; + d.lower = Field(in_e); + EXPECT_TRUE(d.isTrimEligible(e_lo, e_hi)); + // isTrimEligible == false when Q = {date | date ∈ [above, ∞)} (lower-bounded) + d.lower = Field(above); + EXPECT_FALSE(d.isTrimEligible(e_lo, e_hi)); + } + { + // isTrimEligible == true when Q = {date | date ∈ (-∞, in_e]} (upper-bounded) + DateQueryDomain d; + d.predicate_class = TrimPredicateClass::UpperBounded; + d.upper = Field(in_e); + EXPECT_TRUE(d.isTrimEligible(e_lo, e_hi)); + // isTrimEligible == false when Q = {date | date ∈ (-∞, below]} (upper-bounded) + d.upper = Field(below); + EXPECT_FALSE(d.isTrimEligible(e_lo, e_hi)); + } +} + +TEST(TrimMinMaxIndexPhaseC, NormalizeMergesTemporalRange) +{ + auto type = std::make_shared(0); + Attr attr{.col_name = "t", .col_id = 7, .type = type}; + const UInt64 lo = MyDateTime(2020, 1, 1, 0, 0, 0, 0).toPackedUInt(); + const UInt64 hi = MyDateTime(2020, 1, 2, 0, 0, 0, 0).toPackedUInt(); + + auto op = normalizeTemporalRangesForTrim( + createAnd({createGreaterEqual(attr, Field(lo)), createLessEqual(attr, Field(hi))})); + ASSERT_NE(op, nullptr); + EXPECT_EQ(op->name(), "date_range"); + auto reqs = op->getIndexRequests(); + ASSERT_EQ(reqs.size(), 1u); + EXPECT_EQ(reqs[0].preferred_kind, RSIndexKind::PreferTrim); + ASSERT_TRUE(reqs[0].query_domain.has_value()); + EXPECT_EQ(reqs[0].query_domain->predicate_class, TrimPredicateClass::EqualityOrInOrBounded); + + // OR must not rewrite children into PreferTrim DateRange. + auto or_op = createOr({createGreaterEqual(attr, Field(lo)), createEqual(attr, Field(hi))}); + auto normalized_or = normalizeTemporalRangesForTrim(or_op); + EXPECT_EQ(normalized_or->name(), "or"); +} + +// P1-2: unparseable temporal bounds must not be silently dropped into an empty DateRange. +TEST(TrimMinMaxIndexPhaseC, NormalizeKeepsUnparseableTemporalBounds) +{ + auto type = std::make_shared(0); + Attr attr{.col_name = "t", .col_id = 7, .type = type}; + const UInt64 hi = MyDateTime(2020, 1, 2, 0, 0, 0, 0).toPackedUInt(); + + // NULL literal alone: keep original Greater, never emit empty DateRange. + auto null_gt = normalizeTemporalRangesForTrim(createGreater(attr, Field())); + ASSERT_NE(null_gt, nullptr); + EXPECT_EQ(null_gt->name(), "greater"); + + // Negative Int64 is also unparseable as UInt64 bound. + auto neg_gt = normalizeTemporalRangesForTrim(createGreater(attr, Field(static_cast(-1)))); + ASSERT_NE(neg_gt, nullptr); + EXPECT_EQ(neg_gt->name(), "greater"); + + // Mixed: one unparseable bound fails the whole column's DateRange merge. + auto mixed = normalizeTemporalRangesForTrim( + createAnd({createGreaterEqual(attr, Field()), createLessEqual(attr, Field(hi))})); + ASSERT_NE(mixed, nullptr); + EXPECT_EQ(mixed->name(), "and"); + auto and_op = std::dynamic_pointer_cast(mixed); + ASSERT_NE(and_op, nullptr); + ASSERT_EQ(and_op->getChildren().size(), 2u); + EXPECT_EQ(and_op->getChildren()[0]->name(), "greater_equal"); + EXPECT_EQ(and_op->getChildren()[1]->name(), "less_equal"); +} + +// Partial parse failure is per-column: failed column keeps originals; other columns still merge. +TEST(TrimMinMaxIndexPhaseC, NormalizePartialLeafParseFailureIsPerColumn) +{ + auto dt_type = std::make_shared(0); + auto int_type = std::make_shared(); + Attr t{.col_name = "t", .col_id = 7, .type = dt_type}; + Attr u{.col_name = "u", .col_id = 8, .type = dt_type}; + Attr c2{.col_name = "col_2", .col_id = 2, .type = int_type}; + + const UInt64 lo = MyDateTime(2020, 1, 1, 0, 0, 0, 0).toPackedUInt(); + const UInt64 hi = MyDateTime(2020, 1, 2, 0, 0, 0, 0).toPackedUInt(); + + // t >= NULL AND t <= hi -> fail column t + // u >= lo AND u <= hi -> merge column u into DateRange + // col_2 = 1 -> keep non-temporal leaf + auto op = normalizeTemporalRangesForTrim(createAnd({ + createGreaterEqual(t, Field()), + createLessEqual(t, Field(hi)), + createGreaterEqual(u, Field(lo)), + createLessEqual(u, Field(hi)), + createEqual(c2, Field(static_cast(1))), + })); + ASSERT_NE(op, nullptr); + EXPECT_EQ(op->name(), "and"); + auto and_op = std::dynamic_pointer_cast(op); + ASSERT_NE(and_op, nullptr); + + bool found_t_ge = false; + bool found_t_le = false; + bool found_u_date_range = false; + bool found_c2_equal = false; + bool found_t_date_range = false; + + for (const auto & child : and_op->getChildren()) + { + if (child->name() == "greater_equal") + { + auto ids = child->getColumnIDs(); + ASSERT_EQ(ids.size(), 1u); + EXPECT_EQ(ids[0], t.col_id); + found_t_ge = true; + } + else if (child->name() == "less_equal") + { + auto ids = child->getColumnIDs(); + ASSERT_EQ(ids.size(), 1u); + EXPECT_EQ(ids[0], t.col_id); + found_t_le = true; + } + else if (child->name() == "date_range") + { + auto ids = child->getColumnIDs(); + ASSERT_EQ(ids.size(), 1u); + if (ids[0] == t.col_id) + found_t_date_range = true; + else if (ids[0] == u.col_id) + { + found_u_date_range = true; + auto reqs = child->getIndexRequests(); + ASSERT_EQ(reqs.size(), 1u); + ASSERT_TRUE(reqs[0].query_domain.has_value()); + EXPECT_EQ(reqs[0].query_domain->predicate_class, TrimPredicateClass::EqualityOrInOrBounded); + ASSERT_TRUE(reqs[0].query_domain->lower.has_value()); + ASSERT_TRUE(reqs[0].query_domain->upper.has_value()); + EXPECT_EQ(reqs[0].query_domain->lower->safeGet(), lo); + EXPECT_EQ(reqs[0].query_domain->upper->safeGet(), hi); + } + } + else if (child->name() == "equal") + { + auto ids = child->getColumnIDs(); + ASSERT_EQ(ids.size(), 1u); + EXPECT_EQ(ids[0], c2.col_id); + found_c2_equal = true; + } + } + + EXPECT_TRUE(found_t_ge); + EXPECT_TRUE(found_t_le); + EXPECT_FALSE(found_t_date_range); + EXPECT_TRUE(found_u_date_range); + EXPECT_TRUE(found_c2_equal); + EXPECT_EQ(and_op->getChildren().size(), 4u); // t_ge, t_le, u_date_range, c2_equal +} + +// P1-2: empty DateRange domain must return Some, never All. +TEST(TrimMinMaxIndexPhaseC, EmptyDateRangeDomainReturnsSome) +try +{ + auto type = std::make_shared(0); + Attr attr{.col_name = "t", .col_id = 1, .type = type}; + const UInt64 v2021 = MyDateTime(2021, 1, 1, 0, 0, 0, 0).toPackedUInt(); + + auto col = type->createColumn(); + col->insert(Field(v2021)); + MinMaxIndex ordinary(*type); + ordinary.addPack(*col, nullptr); + auto ordinary_ptr = std::make_shared(std::move(ordinary)); + + RSCheckParam param; + param.indexes.emplace(attr.col_id, RSIndex(type, ordinary_ptr)); + + DateQueryDomain empty_domain; + empty_domain.predicate_class = TrimPredicateClass::UpperBounded; // no lower/upper set + auto empty_range = createDateRange(attr, empty_domain); + auto res = empty_range->roughCheck(0, 1, param); + ASSERT_EQ(res.size(), 1u); + EXPECT_EQ(res[0], RSResult::Some); +} +CATCH + +// Design Validation Strategy RSResult matrix: DateRange + Equal/In on the same packs. +TEST(TrimMinMaxIndexPhaseC, RoughCheckCorrectionDesignMatrix) +try +{ + auto type = std::make_shared(0); + Attr attr{.col_name = "t", .col_id = 1, .type = type}; + const UInt64 e_lo = TrimMinMax::defaultLowerBoundPacked(*type); + const UInt64 e_hi = TrimMinMax::defaultUpperBoundPacked(*type); + const UInt64 v2021 = MyDateTime(2021, 1, 1, 0, 0, 0, 0).toPackedUInt(); + const UInt64 v2020 = MyDateTime(2020, 1, 1, 0, 0, 0, 0).toPackedUInt(); + const UInt64 v2022 = MyDateTime(2022, 1, 1, 0, 0, 0, 0).toPackedUInt(); + const UInt64 v2100 = MyDateTime(2100, 1, 1, 0, 0, 0, 0).toPackedUInt(); + const UInt64 v1800 = MyDateTime(1800, 1, 1, 0, 0, 0, 0).toPackedUInt(); + const UInt64 v2021_mid = MyDateTime(2021, 6, 1, 0, 0, 0, 0).toPackedUInt(); + + auto make_col = [&](const std::vector & vals) { + auto col = type->createColumn(); + for (auto v : vals) + col->insert(Field(v)); + return col; + }; + + MinMaxIndex ordinary(*type); + MinMaxIndex trim(*type); + // pack0: {2021, 2100} + TrimMinMax::addOrdinaryAndTrimPack(ordinary, trim, *make_col({v2021, v2100}), nullptr, e_lo, e_hi); + // pack1: {2100} + TrimMinMax::addOrdinaryAndTrimPack(ordinary, trim, *make_col({v2100}), nullptr, e_lo, e_hi); + // pack2: {2021} + TrimMinMax::addOrdinaryAndTrimPack(ordinary, trim, *make_col({v2021}), nullptr, e_lo, e_hi); + // pack3: {1800} + TrimMinMax::addOrdinaryAndTrimPack(ordinary, trim, *make_col({v1800}), nullptr, e_lo, e_hi); + // pack4: {1800, 2100} + TrimMinMax::addOrdinaryAndTrimPack(ordinary, trim, *make_col({v1800, v2100}), nullptr, e_lo, e_hi); + + constexpr size_t pack_count = 5; + auto trim_ptr = std::make_shared(std::move(trim)); + RSCheckParam param; + param.trim_indexes.emplace( + attr.col_id, + TrimRSIndex{ + .type = type, + .minmax = trim_ptr, + .meta = TrimMinMaxIndexMeta{ + .format_version = 1, + .lower_bound = e_lo, + .upper_bound = e_hi, + .pack_count = pack_count}}); + + auto expect_res = [](const RSResults & res, size_t pack, RSResult expected, const char * label) { + ASSERT_LT(pack, res.size()) << label; + EXPECT_EQ(res[pack], expected) << label << " got=" << fmt::format("{}", res[pack]); + if (expected == RSResult::AllNull || expected == RSResult::SomeNull || expected == RSResult::Some) + EXPECT_FALSE(res[pack].allMatch()) << label << " must keep row-level filtering"; + }; + + // --- DateRange bounded: query=[2020, 2022] --- + DateQueryDomain bounded; + bounded.predicate_class = TrimPredicateClass::EqualityOrInOrBounded; + bounded.lower = Field(v2020); + bounded.upper = Field(v2022); + bounded.lower_inclusive = true; + bounded.upper_inclusive = true; + auto range_op = createDateRange(attr, bounded); + auto bounded_res = range_op->roughCheck(0, pack_count, param); + expect_res(bounded_res, 0, RSResult::Some, "DateRange [2020,2022] pack={2021,2100}"); + expect_res(bounded_res, 1, RSResult::None, "DateRange [2020,2022] pack={2100}"); + expect_res(bounded_res, 2, RSResult::All, "DateRange [2020,2022] pack={2021}"); + + // --- DateRange lower-bounded: query>=2020 --- + DateQueryDomain lower_b; + lower_b.predicate_class = TrimPredicateClass::LowerBounded; + lower_b.lower = Field(v2020); + lower_b.lower_inclusive = true; + auto ge_op = createDateRange(attr, lower_b); + auto ge_res = ge_op->roughCheck(0, pack_count, param); + expect_res(ge_res, 1, RSResult::Some, "DateRange >=2020 pack={2100}"); + expect_res(ge_res, 3, RSResult::None, "DateRange >=2020 pack={1800}"); + expect_res(ge_res, 4, RSResult::Some, "DateRange >=2020 pack={1800,2100}"); + + // --- DateRange upper-bounded: query<=2020 --- + DateQueryDomain upper_b; + upper_b.predicate_class = TrimPredicateClass::UpperBounded; + upper_b.upper = Field(v2020); + upper_b.upper_inclusive = true; + auto le_op = createDateRange(attr, upper_b); + auto le_res = le_op->roughCheck(0, pack_count, param); + expect_res(le_res, 3, RSResult::Some, "DateRange <=2020 pack={1800}"); + expect_res(le_res, 1, RSResult::None, "DateRange <=2020 pack={2100}"); + + // --- Equal / In share EqualityOrInOrBounded correction on the same packs --- + auto eq_op = createEqual(attr, Field(v2021)); + auto eq_res = eq_op->roughCheck(0, pack_count, param); + expect_res(eq_res, 0, RSResult::Some, "Equal(2021) pack={2021,2100}"); + expect_res(eq_res, 1, RSResult::None, "Equal(2021) pack={2100}"); + expect_res(eq_res, 2, RSResult::All, "Equal(2021) pack={2021}"); + expect_res(eq_res, 3, RSResult::None, "Equal(2021) pack={1800}"); + expect_res(eq_res, 4, RSResult::None, "Equal(2021) pack={1800,2100}"); + + auto in_op = createIn(attr, {Field(v2021), Field(v2021_mid)}); + auto in_res = in_op->roughCheck(0, pack_count, param); + expect_res(in_res, 0, RSResult::Some, "in(2021,...) pack={2021,2100}"); + expect_res(in_res, 1, RSResult::None, "in(2021,...) pack={2100}"); + expect_res(in_res, 2, RSResult::All, "in(2021,...) pack={2021}"); + expect_res(in_res, 3, RSResult::None, "in(2021,...) pack={1800}"); + expect_res(in_res, 4, RSResult::None, "in(2021,...) pack={1800,2100}"); + + // Single-value IN must match Equal on the shared correction matrix. + auto in_single = createIn(attr, {Field(v2021)}); + EXPECT_EQ(in_single->roughCheck(0, pack_count, param), eq_res); + + // Bounded DateRange and Equal/In agree on shared EqualityOrInOrBounded packs. + EXPECT_EQ(bounded_res[0], eq_res[0]); + EXPECT_EQ(bounded_res[1], eq_res[1]); + EXPECT_EQ(bounded_res[2], eq_res[2]); + + // pack={NULL,2021} must use a Nullable index: empty-value packs on Nullable MinMax + // default to SomeNull (not None), so keep the NULL case on its own index. + { + auto nullable_type = makeNullable(type); + Attr nullable_attr{.col_name = "t", .col_id = 1, .type = nullable_type}; + MinMaxIndex ordinary_n(*nullable_type); + MinMaxIndex trim_n(*nullable_type); + auto col = nullable_type->createColumn(); + col->insertDefault(); + col->insert(Field(v2021)); + TrimMinMax::addOrdinaryAndTrimPack(ordinary_n, trim_n, *col, nullptr, e_lo, e_hi); + EXPECT_TRUE(trim_n.hasNull(0)); + EXPECT_TRUE(trim_n.hasValue(0)); + EXPECT_FALSE(trim_n.hasTrimmedLow(0)); + EXPECT_FALSE(trim_n.hasTrimmedHigh(0)); + + auto trim_n_ptr = std::make_shared(std::move(trim_n)); + RSCheckParam nullable_param; + nullable_param.trim_indexes.emplace( + nullable_attr.col_id, + TrimRSIndex{ + .type = nullable_type, + .minmax = trim_n_ptr, + .meta + = TrimMinMaxIndexMeta{.format_version = 1, .lower_bound = e_lo, .upper_bound = e_hi, .pack_count = 1}}); + + auto null_range = createDateRange(nullable_attr, bounded); + auto null_range_res = null_range->roughCheck(0, 1, nullable_param); + expect_res(null_range_res, 0, RSResult::AllNull, "DateRange [2020,2022] pack={NULL,2021}"); + EXPECT_NE(null_range_res[0], RSResult::All); + + auto null_eq = createEqual(nullable_attr, Field(v2021)); + auto null_eq_res = null_eq->roughCheck(0, 1, nullable_param); + expect_res(null_eq_res, 0, RSResult::AllNull, "Equal(2021) pack={NULL,2021}"); + EXPECT_NE(null_eq_res[0], RSResult::All); + + auto null_in = createIn(nullable_attr, {Field(v2021), Field(v2021_mid)}); + auto null_in_res = null_in->roughCheck(0, 1, nullable_param); + expect_res(null_in_res, 0, RSResult::AllNull, "in(2021,...) pack={NULL,2021}"); + EXPECT_EQ(null_range_res[0], null_eq_res[0]); + EXPECT_EQ(null_eq_res[0], null_in_res[0]); + } +} +CATCH + +// Operator-level "Must fall back" shapes (no SQL): NotEqual / Not(DateRange) / Or(And range, IsNull). +TEST(TrimMinMaxIndexPhaseC, MustFallBackOperatorShapesHaveNoPreferTrim) +{ + auto type = std::make_shared(0); + Attr attr{.col_name = "t", .col_id = 1, .type = type}; + const UInt64 v2020 = MyDateTime(2020, 1, 1, 0, 0, 0, 0).toPackedUInt(); + const UInt64 v2021 = MyDateTime(2021, 1, 1, 0, 0, 0, 0).toPackedUInt(); + const UInt64 e_lo = TrimMinMax::defaultLowerBoundPacked(*type); + const UInt64 e_hi = TrimMinMax::defaultUpperBoundPacked(*type); + + auto has_prefer_trim = [](const RSOperatorPtr & op) { + for (const auto & req : op->getIndexRequests()) + { + if (req.preferred_kind == RSIndexKind::PreferTrim) + return true; + } + return false; + }; + + EXPECT_FALSE(has_prefer_trim(createNotEqual(attr, Field(v2020)))); + + DateQueryDomain bounded; + bounded.predicate_class = TrimPredicateClass::EqualityOrInOrBounded; + bounded.lower = Field(v2020); + bounded.upper = Field(v2021); + bounded.lower_inclusive = true; + bounded.upper_inclusive = true; + auto date_range = createDateRange(attr, bounded); + EXPECT_TRUE(has_prefer_trim(date_range)); + EXPECT_FALSE(has_prefer_trim(createNot(date_range))); + + // BETWEEN under OR is not rewritten to DateRange; GE/LE themselves are Normal. + auto between_under_or = createOr( + {createAnd({createGreaterEqual(attr, Field(v2020)), createLessEqual(attr, Field(v2021))}), createIsNull(attr)}); + EXPECT_FALSE(has_prefer_trim(between_under_or)); + EXPECT_EQ(normalizeTemporalRangesForTrim(between_under_or)->name(), "or"); + EXPECT_FALSE(has_prefer_trim(normalizeTemporalRangesForTrim(between_under_or))); + + // Out-of-E one-sided range remains PreferTrim at request time but is ineligible. + DateQueryDomain out_e; + out_e.predicate_class = TrimPredicateClass::LowerBounded; + out_e.lower = Field(MyDateTime(2200, 1, 1, 0, 0, 0, 0).toPackedUInt()); + out_e.lower_inclusive = true; + auto out_range = createDateRange(attr, out_e); + ASSERT_TRUE(has_prefer_trim(out_range)); + ASSERT_TRUE(out_range->getIndexRequests()[0].query_domain.has_value()); + EXPECT_FALSE(out_range->getIndexRequests()[0].query_domain->isTrimEligible(e_lo, e_hi)); +} + +// NOT / NOT IN are outside trim support. Not must not PreferTrim, and must not let an empty +// trim pack turn In(..., NULL) into None then !None => All (skipping row filters). +TEST(TrimMinMaxIndexPhaseC, NotInWithNullDoesNotUseTrimToAdvertiseAll) +try +{ + auto type = std::make_shared(0); + Attr attr{.col_name = "t", .col_id = 1, .type = type}; + const UInt64 e_lo = TrimMinMax::defaultLowerBoundPacked(*type); + const UInt64 e_hi = TrimMinMax::defaultUpperBoundPacked(*type); + const UInt64 v2020 = MyDateTime(2020, 1, 1, 0, 0, 0, 0).toPackedUInt(); + const UInt64 v2200 = MyDateTime(2200, 1, 1, 0, 0, 0, 0).toPackedUInt(); + + auto make_col = [&](const std::vector & vals) { + auto col = type->createColumn(); + for (auto v : vals) + col->insert(Field(v)); + return col; + }; + + MinMaxIndex ordinary(*type); + MinMaxIndex trim(*type); + // pack0: only out-of-E value β€” trim has_value=false, has_trimmed_high + TrimMinMax::addOrdinaryAndTrimPack(ordinary, trim, *make_col({v2200}), nullptr, e_lo, e_hi); + EXPECT_FALSE(trim.hasValue(0)); + EXPECT_TRUE(trim.hasTrimmedHigh(0)); + + auto ordinary_ptr = std::make_shared(std::move(ordinary)); + auto trim_ptr = std::make_shared(std::move(trim)); + + RSCheckParam param; + param.indexes.emplace(attr.col_id, RSIndex(type, ordinary_ptr)); + param.trim_indexes.emplace( + attr.col_id, + TrimRSIndex{ + .type = type, + .minmax = trim_ptr, + .meta + = TrimMinMaxIndexMeta{.format_version = 1, .lower_bound = e_lo, .upper_bound = e_hi, .pack_count = 1}}); + + auto not_in_with_null = createNot(createIn(attr, {Field(v2020), Field()})); + + // Not must demote PreferTrim from In to Normal. + auto reqs = not_in_with_null->getIndexRequests(); + ASSERT_EQ(reqs.size(), 1u); + EXPECT_EQ(reqs[0].preferred_kind, RSIndexKind::Normal); + EXPECT_FALSE(reqs[0].query_domain.has_value()); + + // Ordinary semantics: In(2020, NULL) on {2200} => NoneNull, Not => AllNull (not All). + // AllNull does not allMatch(), so row-level filter is retained. + auto ordinary_only = param; + ordinary_only.trim_indexes.clear(); + auto ordinary_res = not_in_with_null->roughCheck(0, 1, ordinary_only); + ASSERT_EQ(ordinary_res.size(), 1u); + EXPECT_EQ(ordinary_res[0], RSResult::AllNull); + EXPECT_FALSE(ordinary_res[0].allMatch()); + + // Even when trim is already loaded (sibling PreferTrim), Not must not advertise All. + auto with_trim = not_in_with_null->roughCheck(0, 1, param); + ASSERT_EQ(with_trim.size(), 1u); + EXPECT_EQ(with_trim[0], RSResult::AllNull); + EXPECT_NE(with_trim[0], RSResult::All); + EXPECT_FALSE(with_trim[0].allMatch()); + + // Without NULL, Not In(2020) on {2200} may correctly be All (all rows match). + auto not_in = createNot(createIn(attr, {Field(v2020)})); + auto not_in_res = not_in->roughCheck(0, 1, param); + ASSERT_EQ(not_in_res.size(), 1u); + EXPECT_EQ(not_in_res[0], RSResult::All); +} +CATCH + +// P1-1: trim eligibility is per-predicate, not per-column. +// pack={2200}, predicate: col=2020 OR col=2200 must not be None. +TEST(TrimMinMaxIndexPhaseC, OrDoesNotShareTrimEligibilityAcrossPredicates) +try +{ + auto type = std::make_shared(0); + Attr attr{.col_name = "t", .col_id = 1, .type = type}; + const UInt64 e_lo = TrimMinMax::defaultLowerBoundPacked(*type); + const UInt64 e_hi = TrimMinMax::defaultUpperBoundPacked(*type); + const UInt64 v2020 = MyDateTime(2020, 1, 1, 0, 0, 0, 0).toPackedUInt(); + const UInt64 v2200 = MyDateTime(2200, 1, 1, 0, 0, 0, 0).toPackedUInt(); + + auto make_col = [&](const std::vector & vals) { + auto col = type->createColumn(); + for (auto v : vals) + col->insert(Field(v)); + return col; + }; + + MinMaxIndex ordinary(*type); + MinMaxIndex trim(*type); + // pack0: {2200} only β€” trim has no in-range value, has_trimmed_high + TrimMinMax::addOrdinaryAndTrimPack(ordinary, trim, *make_col({v2200}), nullptr, e_lo, e_hi); + // pack1: {2020} only + TrimMinMax::addOrdinaryAndTrimPack(ordinary, trim, *make_col({v2020}), nullptr, e_lo, e_hi); + + auto ordinary_ptr = std::make_shared(std::move(ordinary)); + auto trim_ptr = std::make_shared(std::move(trim)); + RSCheckParam param; + param.indexes.emplace(attr.col_id, RSIndex(type, ordinary_ptr)); + param.trim_indexes.emplace( + attr.col_id, + TrimRSIndex{ + .type = type, + .minmax = trim_ptr, + .meta + = TrimMinMaxIndexMeta{.format_version = 1, .lower_bound = e_lo, .upper_bound = e_hi, .pack_count = 2}}); + + auto eq_in_e = createEqual(attr, Field(v2020)); + auto eq_out_e = createEqual(attr, Field(v2200)); + + // Non-eligible Equal must ignore the column's loaded trim and use ordinary. + auto out_res = eq_out_e->roughCheck(0, 2, param); + EXPECT_EQ(out_res[0], RSResult::All); // {2200} matches + EXPECT_EQ(out_res[1], RSResult::None); // {2020} + + // Eligible Equal may use trim. + auto in_res = eq_in_e->roughCheck(0, 2, param); + EXPECT_EQ(in_res[0], RSResult::None); // {2200} trimmed out + EXPECT_EQ(in_res[1], RSResult::All); // {2020} + + const RSOperators or_ops = { + createOr({eq_in_e, eq_out_e}), + createOr({eq_out_e, eq_in_e}), + }; + for (const auto & or_op : or_ops) + { + auto or_res = or_op->roughCheck(0, 2, param); + EXPECT_NE(or_res[0], RSResult::None); // must keep pack with 2200 + EXPECT_NE(or_res[1], RSResult::None); // must keep pack with 2020 + } + + // IN containing an out-of-E value is not trim-eligible either. + auto in_mixed = createIn(attr, {Field(v2020), Field(v2200)}); + auto in_mixed_res = in_mixed->roughCheck(0, 2, param); + EXPECT_EQ(in_mixed_res[0], RSResult::All); + EXPECT_EQ(in_mixed_res[1], RSResult::All); +} +CATCH + +TEST_F(TrimMinMaxIndexWriteTest, PackFilterUsesTrimWhenReadEnabled) +try +{ + auto dm_file = writeDMFile(makeBlockWithOutlier(/*rows*/ 8), /*enable_trim_write*/ true); + ASSERT_TRUE(dm_file->getColumnStat(settle_col_id).trim_minmax_index.has_value()); + + auto type = std::make_shared(0); + Attr attr{.col_name = "settle_time", .col_id = settle_col_id, .type = type}; + const UInt64 lo = MyDateTime(2020, 1, 1, 0, 0, 0, 0).toPackedUInt(); + const UInt64 hi = MyDateTime(2021, 1, 1, 0, 0, 0, 0).toPackedUInt(); + DateQueryDomain domain; + domain.predicate_class = TrimPredicateClass::EqualityOrInOrBounded; + domain.lower = Field(lo); + domain.upper = Field(hi); + auto filter = createDateRange(attr, domain); + + auto scan_context = std::make_shared(); + auto pack_result = DMFilePackFilter::loadFrom( + /*index_cache*/ nullptr, + file_provider, + /*read_limiter*/ nullptr, + scan_context, + dm_file, + /*set_cache_if_miss*/ false, + /*rowkey_ranges*/ {}, + filter, + /*read_packs*/ {}, + /*tracing_id*/ "trim_phase_c", + /*enable_trim_minmax*/ true, + ReadTag::Query); + + // Sentinel-only values are trimmed out of min-max; bounded query must not be All. + const auto & pack_res = pack_result->getPackRes(); + ASSERT_FALSE(pack_res.empty()); + for (auto r : pack_res) + EXPECT_NE(r, RSResult::All); +} +CATCH + +// P1-1 end-to-end: after an in-E Equal loads trim, an out-of-E Equal under OR must +// still load ordinary and keep packs that only contain the out-of-E value. +TEST_F(TrimMinMaxIndexWriteTest, PackFilterOrMixedEligibilityLoadsOrdinary) +try +{ + auto type = std::make_shared(0); + Block block = DMTestEnv::prepareSimpleWriteBlock(0, 8, /*reversed*/ false); + auto col = type->createColumn(); + const UInt64 out_e = MyDateTime(2200, 1, 1, 0, 0, 0, 0).toPackedUInt(); + for (size_t i = 0; i < 8; ++i) + col->insert(Field(out_e)); + block.insert(ColumnWithTypeAndName(std::move(col), type, "settle_time", settle_col_id)); + + auto dm_file = writeDMFile(block, /*enable_trim_write*/ true); + ASSERT_TRUE(dm_file->getColumnStat(settle_col_id).trim_minmax_index.has_value()); + + Attr attr{.col_name = "settle_time", .col_id = settle_col_id, .type = type}; + const UInt64 in_e = MyDateTime(2020, 1, 1, 0, 0, 0, 0).toPackedUInt(); + + const RSOperators filters = { + createOr({createEqual(attr, Field(in_e)), createEqual(attr, Field(out_e))}), + createOr({createEqual(attr, Field(out_e)), createEqual(attr, Field(in_e))}), + }; + for (const auto & filter : filters) + { + auto scan_context = std::make_shared(); + auto pack_result = DMFilePackFilter::loadFrom( + /*index_cache*/ nullptr, + file_provider, + /*read_limiter*/ nullptr, + scan_context, + dm_file, + /*set_cache_if_miss*/ false, + /*rowkey_ranges*/ {}, + filter, + /*read_packs*/ {}, + /*tracing_id*/ "trim_phase_c_or", + /*enable_trim_minmax*/ true, + ReadTag::Query); + + const auto & pack_res = pack_result->getPackRes(); + ASSERT_FALSE(pack_res.empty()); + for (auto r : pack_res) + EXPECT_NE(r, RSResult::None) << "out-of-E OR branch must keep packs with 2200"; + } +} +CATCH + +// Attr.type may be empty when column id is missing from table defines. +// Equal/In must not dereference it in getIndexRequests (called unconditionally). +TEST(TrimMinMaxIndexSafety, EqualInNullAttrTypeDoesNotCrash) +{ + Attr attr{.col_name = "", .col_id = 42, .type = DataTypePtr{}}; + auto eq = createEqual(attr, Field(static_cast(1))); + auto in = createIn(attr, {Field(static_cast(1)), Field(static_cast(2))}); + + auto eq_reqs = eq->getIndexRequests(); + ASSERT_EQ(eq_reqs.size(), 1u); + EXPECT_EQ(eq_reqs[0].preferred_kind, RSIndexKind::Normal); + EXPECT_FALSE(eq_reqs[0].query_domain.has_value()); + + auto in_reqs = in->getIndexRequests(); + ASSERT_EQ(in_reqs.size(), 1u); + EXPECT_EQ(in_reqs[0].preferred_kind, RSIndexKind::Normal); + EXPECT_FALSE(in_reqs[0].query_domain.has_value()); + + RSCheckParam param; + auto eq_res = eq->roughCheck(0, 1, param); + ASSERT_EQ(eq_res.size(), 1u); + EXPECT_EQ(eq_res[0], RSResult::Some); + + auto in_res = in->roughCheck(0, 1, param); + ASSERT_EQ(in_res.size(), 1u); + EXPECT_EQ(in_res[0], RSResult::Some); +} + +} // namespace DB::DM::tests diff --git a/dbms/src/Storages/DeltaMerge/Segment.cpp b/dbms/src/Storages/DeltaMerge/Segment.cpp index 633383abdf8..609f9ae4825 100644 --- a/dbms/src/Storages/DeltaMerge/Segment.cpp +++ b/dbms/src/Storages/DeltaMerge/Segment.cpp @@ -1032,13 +1032,16 @@ BlockInputStreamPtr Segment::getInputStream( pack_filter_results.reserve(segment_snap->stable->getDMFiles().size()); for (const auto & dmfile : segment_snap->stable->getDMFiles()) { + // Normal mode is the query read path; other modes build MVCC bitmaps. + const auto pack_filter_read_tag = (read_mode == ReadMode::Normal) ? ReadTag::Query : ReadTag::MVCC; auto result = DMFilePackFilter::loadFrom( dm_context, dmfile, /*set_cache_if_miss*/ true, real_ranges, executor ? executor->rs_operator : EMPTY_RS_OPERATOR, - /*read_pack*/ {}); + /*read_pack*/ {}, + pack_filter_read_tag); pack_filter_results.push_back(result); } diff --git a/dbms/src/Storages/DeltaMerge/dtpb/dmfile.proto b/dbms/src/Storages/DeltaMerge/dtpb/dmfile.proto index 8191bc7a370..9eaab6c7a2e 100644 --- a/dbms/src/Storages/DeltaMerge/dtpb/dmfile.proto +++ b/dbms/src/Storages/DeltaMerge/dtpb/dmfile.proto @@ -70,6 +70,20 @@ message ColumnStat { optional VectorIndexFilePropsV1Deprecated deprecated_vector_index = 102; // deprecated. See field 104. repeated VectorIndexFilePropsV1Deprecated deprecated_vector_indexes = 103; // deprecated. See field 104. repeated DMFileIndexInfo indexes = 104; + + // DMFile-internal trim min-max index. At most one per column. + // Intentionally not part of `indexes = 104` so old Readers treat this as an + // unknown field and skip integrityCheckIndexInfoV2 / IndexFileKind checks. + optional TrimMinMaxIndexProps trim_minmax_index = 105; +} + +// Pack-level trim min-max for DATE / DATETIME / TIMESTAMP columns. +// Bounds use the column nested type's packed encoding; V1 semantics are [lower, upper). +message TrimMinMaxIndexProps { + optional uint32 format_version = 1; + optional bytes lower_bound = 2; + optional bytes upper_bound = 3; + optional uint64 pack_count = 4; } message DMFileIndexInfo { diff --git a/dbms/src/Storages/StorageDisaggregatedColumnar.cpp b/dbms/src/Storages/StorageDisaggregatedColumnar.cpp index d0e1657a8ae..e5918722543 100644 --- a/dbms/src/Storages/StorageDisaggregatedColumnar.cpp +++ b/dbms/src/Storages/StorageDisaggregatedColumnar.cpp @@ -129,6 +129,7 @@ struct RNColumnarReaderSharedContext String table_info_data; String ann_query_info_data; String fts_query_info_data; + bool enable_trim_minmax = false; RaftStoreProxyPtr proxy_ptr{}; ClearSharedSnapAccessByStartTsFn clear_shared_snap_access_by_start_ts = nullptr; std::shared_ptr output_lock = std::make_shared(); @@ -470,6 +471,7 @@ std::shared_ptr buildColumnarReaderSharedContext( shared_context->table_info_data = table_info.SerializeAsString(); shared_context->ann_query_info_data = table_scan.getANNQueryInfo().SerializeAsString(); shared_context->fts_query_info_data = table_scan.getFTSQueryInfo().SerializeAsString(); + shared_context->enable_trim_minmax = context.getSettingsRef().dt_enable_trim_minmax; return shared_context; } @@ -764,6 +766,7 @@ ColumnarReaderPtr createColumnarReader( std::move(filter_conditions_view), std::move(ann_query_info_view), std::move(fts_query_info_view), + shared_context.enable_trim_minmax, proxy_helper->proxy_ptr); bool reader_returned = false; SCOPE_EXIT({ diff --git a/dbms/src/Storages/StorageDisaggregatedRemote.cpp b/dbms/src/Storages/StorageDisaggregatedRemote.cpp index d796683228f..d35677fe2aa 100644 --- a/dbms/src/Storages/StorageDisaggregatedRemote.cpp +++ b/dbms/src/Storages/StorageDisaggregatedRemote.cpp @@ -626,8 +626,13 @@ std::tuple StorageDisaggregated::buildRSO std::vector{}, 0, db_context.getTimezoneInfo()); - const auto rs_operator - = DM::RSOperator::build(dag_query, table_scan.getColumns(), *columns_to_read, enable_rs_filter, log); + const auto rs_operator = DM::RSOperator::build( + dag_query, + table_scan.getColumns(), + *columns_to_read, + enable_rs_filter, + log, + db_context.getSettingsRef().dt_enable_trim_minmax); const auto & used_indexes = dag_query->used_indexes; // build column_range diff --git a/dbms/src/Storages/tests/gtest_filter_parser.cpp b/dbms/src/Storages/tests/gtest_filter_parser.cpp index bc2882b1f47..3ae29fa878f 100644 --- a/dbms/src/Storages/tests/gtest_filter_parser.cpp +++ b/dbms/src/Storages/tests/gtest_filter_parser.cpp @@ -12,6 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. +#include +#include #include #include #include @@ -24,9 +26,13 @@ #include #include #include +#include +#include +#include #include #include #include +#include #include #include #include @@ -65,7 +71,11 @@ class FilterParserTest : public ::testing::Test LoggerPtr log; ContextPtr ctx; static TimezoneInfo default_timezone_info; - DM::RSOperatorPtr generateRsOperator(String table_info_json, const String & query, TimezoneInfo & timezone_info); + DM::RSOperatorPtr generateRsOperator( + String table_info_json, + const String & query, + TimezoneInfo & timezone_info = default_timezone_info, + bool enable_trim_minmax = false); }; TimezoneInfo FilterParserTest::default_timezone_info; @@ -73,7 +83,8 @@ TimezoneInfo FilterParserTest::default_timezone_info; DM::RSOperatorPtr FilterParserTest::generateRsOperator( const String table_info_json, const String & query, - TimezoneInfo & timezone_info = default_timezone_info) + TimezoneInfo & timezone_info, + bool enable_trim_minmax) { const TiDB::TableInfo table_info(table_info_json, NullspaceID); @@ -125,7 +136,7 @@ DM::RSOperatorPtr FilterParserTest::generateRsOperator( column_id_to_attr[cd.id] = DM::Attr{.col_name = cd.name, .col_id = cd.id, .type = cd.type}; } - return DM::FilterParser::parseDAGQuery(*dag_query, table_info.columns, column_id_to_attr, log); + return DM::FilterParser::parseDAGQuery(*dag_query, table_info.columns, column_id_to_attr, log, enable_trim_minmax); } // Test cases for col and literal @@ -612,17 +623,26 @@ try auto ctx = TiFlashTestEnv::getContext(); auto & timezone_info = ctx->getTimezoneInfo(); convertTimeZone(origin_time_stamp, converted_time, *timezone_info.timezone, time_zone_utc); + const auto query = String("select * from default.t_111 where col_timestamp > cast_string_datetime('") + datetime + + String("')"); - auto rs_operator = generateRsOperator( - table_info_json, - fmt::format("select * from default.t_111 where col_timestamp > cast_string_datetime('{}')", datetime), - timezone_info); + auto rs_operator = generateRsOperator(table_info_json, query, timezone_info, /*enable_trim_minmax*/ false); EXPECT_EQ(rs_operator->name(), "greater"); EXPECT_EQ(rs_operator->getColumnIDs().size(), 1); EXPECT_EQ(rs_operator->getColumnIDs()[0], 4); EXPECT_EQ( rs_operator->toDebugString(), - fmt::format(R"json({{"op":"greater","col":"col_timestamp","value":"{}"}})json", converted_time)); + fmt::format(R"json({{"op":"greater","col":"col_timestamp","value":"{}"}})json", toString(converted_time))); + + rs_operator = generateRsOperator(table_info_json, query, timezone_info, /*enable_trim_minmax*/ true); + EXPECT_EQ(rs_operator->name(), "date_range"); + EXPECT_EQ(rs_operator->getColumnIDs().size(), 1); + EXPECT_EQ(rs_operator->getColumnIDs()[0], 4); + EXPECT_EQ( + rs_operator->toDebugString(), + fmt::format( + R"json({{"op":"date_range","col":"col_timestamp","class":"LowerBounded","lower":"{}","lower_inclusive":false,"upper":"","upper_inclusive":true}})json", + converted_time)); auto sets = rs_operator->buildSets(used_indexes); EXPECT_TRUE(sets != nullptr); EXPECT_EQ(sets->toDebugString(), fmt::format("4: [{}, 18446744073709551615]", converted_time + 1)); @@ -634,39 +654,57 @@ try auto & timezone_info = ctx->getTimezoneInfo(); timezone_info.resetByTimezoneName("America/Chicago"); convertTimeZone(origin_time_stamp, converted_time, *timezone_info.timezone, time_zone_utc); + const auto query = String("select * from default.t_111 where col_timestamp > cast_string_datetime('") + datetime + + String("')"); - auto rs_operator = generateRsOperator( - table_info_json, - fmt::format("select * from default.t_111 where col_timestamp > cast_string_datetime('{}')", datetime), - timezone_info); + auto rs_operator = generateRsOperator(table_info_json, query, timezone_info, /*enable_trim_minmax*/ false); EXPECT_EQ(rs_operator->name(), "greater"); EXPECT_EQ(rs_operator->getColumnIDs().size(), 1); EXPECT_EQ(rs_operator->getColumnIDs()[0], 4); EXPECT_EQ( rs_operator->toDebugString(), - fmt::format(R"json({{"op":"greater","col":"col_timestamp","value":"{}"}})json", converted_time)); + fmt::format(R"json({{"op":"greater","col":"col_timestamp","value":"{}"}})json", toString(converted_time))); + + rs_operator = generateRsOperator(table_info_json, query, timezone_info, /*enable_trim_minmax*/ true); + EXPECT_EQ(rs_operator->name(), "date_range"); + EXPECT_EQ(rs_operator->getColumnIDs().size(), 1); + EXPECT_EQ(rs_operator->getColumnIDs()[0], 4); + EXPECT_EQ( + rs_operator->toDebugString(), + fmt::format( + R"json({{"op":"date_range","col":"col_timestamp","class":"LowerBounded","lower":"{}","lower_inclusive":false,"upper":"","upper_inclusive":true}})json", + converted_time)); auto sets = rs_operator->buildSets(used_indexes); EXPECT_TRUE(sets != nullptr); EXPECT_EQ(sets->toDebugString(), fmt::format("4: [{}, 18446744073709551615]", converted_time + 1)); } { - // Greater between TimeStamp col and Datetime literal, use Chicago timezone + // Greater between TimeStamp col and Datetime literal, use timezone offset auto ctx = TiFlashTestEnv::getContext(); auto & timezone_info = ctx->getTimezoneInfo(); timezone_info.resetByTimezoneOffset(28800); convertTimeZoneByOffset(origin_time_stamp, converted_time, false, timezone_info.timezone_offset); + const auto query = String("select * from default.t_111 where col_timestamp > cast_string_datetime('") + datetime + + String("')"); - auto rs_operator = generateRsOperator( - table_info_json, - fmt::format("select * from default.t_111 where col_timestamp > cast_string_datetime('{}')", datetime), - timezone_info); + auto rs_operator = generateRsOperator(table_info_json, query, timezone_info, /*enable_trim_minmax*/ false); EXPECT_EQ(rs_operator->name(), "greater"); EXPECT_EQ(rs_operator->getColumnIDs().size(), 1); EXPECT_EQ(rs_operator->getColumnIDs()[0], 4); EXPECT_EQ( rs_operator->toDebugString(), - fmt::format(R"json({{"op":"greater","col":"col_timestamp","value":"{}"}})json", converted_time)); + fmt::format(R"json({{"op":"greater","col":"col_timestamp","value":"{}"}})json", toString(converted_time))); + + rs_operator = generateRsOperator(table_info_json, query, timezone_info, /*enable_trim_minmax*/ true); + EXPECT_EQ(rs_operator->name(), "date_range"); + EXPECT_EQ(rs_operator->getColumnIDs().size(), 1); + EXPECT_EQ(rs_operator->getColumnIDs()[0], 4); + EXPECT_EQ( + rs_operator->toDebugString(), + fmt::format( + R"json({{"op":"date_range","col":"col_timestamp","class":"LowerBounded","lower":"{}","lower_inclusive":false,"upper":"","upper_inclusive":true}})json", + converted_time)); auto sets = rs_operator->buildSets(used_indexes); EXPECT_TRUE(sets != nullptr); EXPECT_EQ(sets->toDebugString(), fmt::format("4: [{}, 18446744073709551615]", converted_time + 1)); @@ -674,15 +712,29 @@ try { // Greater between Datetime col and Datetime literal - auto rs_operator = generateRsOperator( - table_info_json, - fmt::format("select * from default.t_111 where col_datetime > cast_string_datetime('{}')", datetime)); + const auto query + = fmt::format("select * from default.t_111 where col_datetime > cast_string_datetime('{}')", datetime); + + auto rs_operator + = generateRsOperator(table_info_json, query, default_timezone_info, /*enable_trim_minmax*/ false); EXPECT_EQ(rs_operator->name(), "greater"); EXPECT_EQ(rs_operator->getColumnIDs().size(), 1); EXPECT_EQ(rs_operator->getColumnIDs()[0], 5); EXPECT_EQ( rs_operator->toDebugString(), - fmt::format(R"json({{"op":"greater","col":"col_datetime","value":"{}"}})json", origin_time_stamp)); + fmt::format( + R"json({{"op":"greater","col":"col_datetime","value":"{}"}})json", + toString(origin_time_stamp))); + + rs_operator = generateRsOperator(table_info_json, query, default_timezone_info, /*enable_trim_minmax*/ true); + EXPECT_EQ(rs_operator->name(), "date_range"); + EXPECT_EQ(rs_operator->getColumnIDs().size(), 1); + EXPECT_EQ(rs_operator->getColumnIDs()[0], 5); + EXPECT_EQ( + rs_operator->toDebugString(), + fmt::format( + R"json({{"op":"date_range","col":"col_datetime","class":"LowerBounded","lower":"{}","lower_inclusive":false,"upper":"","upper_inclusive":true}})json", + origin_time_stamp)); auto sets = rs_operator->buildSets(used_indexes); EXPECT_TRUE(sets != nullptr); EXPECT_EQ(sets->toDebugString(), fmt::format("5: [{}, 18446744073709551615]", origin_time_stamp + 1)); @@ -690,15 +742,27 @@ try { // Greater between Date col and Datetime literal - auto rs_operator = generateRsOperator( - table_info_json, - fmt::format("select * from default.t_111 where col_date > cast_string_datetime('{}')", datetime)); + const auto query + = fmt::format("select * from default.t_111 where col_date > cast_string_datetime('{}')", datetime); + + auto rs_operator + = generateRsOperator(table_info_json, query, default_timezone_info, /*enable_trim_minmax*/ false); EXPECT_EQ(rs_operator->name(), "greater"); EXPECT_EQ(rs_operator->getColumnIDs().size(), 1); EXPECT_EQ(rs_operator->getColumnIDs()[0], 6); EXPECT_EQ( rs_operator->toDebugString(), - fmt::format(R"json({{"op":"greater","col":"col_date","value":"{}"}})json", origin_time_stamp)); + fmt::format(R"json({{"op":"greater","col":"col_date","value":"{}"}})json", toString(origin_time_stamp))); + + rs_operator = generateRsOperator(table_info_json, query, default_timezone_info, /*enable_trim_minmax*/ true); + EXPECT_EQ(rs_operator->name(), "date_range"); + EXPECT_EQ(rs_operator->getColumnIDs().size(), 1); + EXPECT_EQ(rs_operator->getColumnIDs()[0], 6); + EXPECT_EQ( + rs_operator->toDebugString(), + fmt::format( + R"json({{"op":"date_range","col":"col_date","class":"LowerBounded","lower":"{}","lower_inclusive":false,"upper":"","upper_inclusive":true}})json", + origin_time_stamp)); auto sets = rs_operator->buildSets(used_indexes); EXPECT_TRUE(sets != nullptr); EXPECT_EQ(sets->toDebugString(), fmt::format("6: [{}, 18446744073709551615]", origin_time_stamp + 1)); @@ -706,6 +770,435 @@ try } CATCH +// Temporal-Type Tests: TIMESTAMP DST vs standard time, and DATE PreferTrim equality. +TEST_F(FilterParserTest, TimestampTrimNormalizeDSTAndDateEqual) +try +{ + const String table_info_json = R"json({ + "cols":[ + {"comment":"","default":null,"default_bit":null,"id":4,"name":{"L":"col_timestamp","O":"col_time"},"offset":-1,"origin_default":null,"state":0,"type":{"Charset":null,"Collate":null,"Decimal":6,"Elems":null,"Flag":1,"Flen":0,"Tp":7}}, + {"comment":"","default":null,"default_bit":null,"id":6,"name":{"L":"col_date","O":"col_date"},"offset":-1,"origin_default":null,"state":0,"type":{"Charset":null,"Collate":null,"Decimal":0,"Elems":null,"Flag":1,"Flen":0,"Tp":14}} + ], + "pk_is_handle":false,"index_info":[],"is_common_handle":false, + "name":{"L":"t_111","O":"t_111"},"partition":null, + "comment":"Mocked.","id":30,"schema_version":-1,"state":0,"tiflash_replica":{"Count":0},"update_timestamp":1636471547239654 +})json"; + + const auto & time_zone_utc = DateLUT::instance("UTC"); + auto ctx = TiFlashTestEnv::getContext(); + auto & timezone_info = ctx->getTimezoneInfo(); + timezone_info.resetByTimezoneName("America/Chicago"); + + auto parse_local = [](const String & datetime) { + ReadBufferFromMemory read_buffer(datetime.c_str(), datetime.size()); + UInt64 origin = 0; + EXPECT_TRUE(tryReadMyDateTimeText(origin, 6, read_buffer)); + return origin; + }; + + const String winter = "2021-01-15 12:00:00.000000"; // CST = UTC-6 + const String summer = "2021-07-15 12:00:00.000000"; // CDT = UTC-5 + const UInt64 winter_local = parse_local(winter); + const UInt64 summer_local = parse_local(summer); + + UInt64 winter_chicago_utc = 0; + UInt64 summer_chicago_utc = 0; + convertTimeZone(winter_local, winter_chicago_utc, *timezone_info.timezone, time_zone_utc); + convertTimeZone(summer_local, summer_chicago_utc, *timezone_info.timezone, time_zone_utc); + + // Fixed CST offset (-6h): matches Chicago in winter, differs in summer under DST. + constexpr Int64 cst_offset = -6 * 3600; + UInt64 winter_cst_utc = 0; + UInt64 summer_cst_utc = 0; + convertTimeZoneByOffset(winter_local, winter_cst_utc, /*from_utc*/ false, cst_offset); + convertTimeZoneByOffset(summer_local, summer_cst_utc, /*from_utc*/ false, cst_offset); + EXPECT_EQ(winter_chicago_utc, winter_cst_utc); + EXPECT_NE(summer_chicago_utc, summer_cst_utc); + + for (const auto & [label, datetime, expected_utc] : + {std::tuple{"winter", winter, winter_chicago_utc}, + {"summer", summer, summer_chicago_utc}}) + { + // Use >= so normalize emits inclusive LowerBounded DateRange. + const auto query = String("select * from default.t_111 where col_timestamp >= cast_string_datetime('") + + datetime + String("')"); + auto rs_operator = generateRsOperator(table_info_json, query, timezone_info, /*enable_trim_minmax*/ true); + EXPECT_EQ(rs_operator->name(), "date_range") << label; + auto reqs = rs_operator->getIndexRequests(); + ASSERT_EQ(reqs.size(), 1u) << label; + EXPECT_EQ(reqs[0].preferred_kind, DM::RSIndexKind::PreferTrim) << label; + ASSERT_TRUE(reqs[0].query_domain.has_value()) << label; + EXPECT_EQ(reqs[0].query_domain->predicate_class, DM::TrimPredicateClass::LowerBounded) << label; + ASSERT_TRUE(reqs[0].query_domain->lower.has_value()) << label; + EXPECT_EQ(reqs[0].query_domain->lower->safeGet(), expected_utc) << label; + EXPECT_TRUE(reqs[0].query_domain->lower_inclusive) << label; + } + + // DATE equality -> PreferTrim (no timezone conversion). + { + const String date_lit = "2020-01-01"; + const auto query + = fmt::format("select * from default.t_111 where col_date = cast_string_datetime('{}')", date_lit); + auto rs_operator + = generateRsOperator(table_info_json, query, default_timezone_info, /*enable_trim_minmax*/ true); + EXPECT_EQ(rs_operator->name(), "equal"); + auto reqs = rs_operator->getIndexRequests(); + ASSERT_EQ(reqs.size(), 1u); + EXPECT_EQ(reqs[0].preferred_kind, DM::RSIndexKind::PreferTrim); + ASSERT_TRUE(reqs[0].query_domain.has_value()); + EXPECT_EQ(reqs[0].query_domain->predicate_class, DM::TrimPredicateClass::EqualityOrInOrBounded); + } +} +CATCH + +// normalizeTemporalRangesForTrim via FilterParser (gated by enable_trim_minmax). +TEST_F(FilterParserTest, NormalizeTemporalRangesForTrim) +try +{ + const String table_info_json = R"json({ + "cols":[ + {"comment":"","default":null,"default_bit":null,"id":2,"name":{"L":"col_2","O":"col_2"},"offset":-1,"origin_default":null,"state":0,"type":{"Charset":null,"Collate":null,"Decimal":0,"Elems":null,"Flag":4097,"Flen":0,"Tp":8}}, + {"comment":"","default":null,"default_bit":null,"id":5,"name":{"L":"col_datetime","O":"col_datetime"},"offset":-1,"origin_default":null,"state":0,"type":{"Charset":null,"Collate":null,"Decimal":5,"Elems":null,"Flag":1,"Flen":0,"Tp":12}} + ], + "pk_is_handle":false,"index_info":[],"is_common_handle":false, + "name":{"L":"t_111","O":"t_111"},"partition":null, + "comment":"Mocked.","id":30,"schema_version":-1,"state":0,"tiflash_replica":{"Count":0},"update_timestamp":1636471547239654 +})json"; + + const String lo_str = "2021-10-26 17:00:00.00000"; + const String hi_str = "2021-10-27 17:00:00.00000"; + UInt64 lo = 0; + UInt64 hi = 0; + { + ReadBufferFromMemory lo_buf(lo_str.c_str(), lo_str.size()); + ASSERT_TRUE(tryReadMyDateTimeText(lo, 6, lo_buf)); + ReadBufferFromMemory hi_buf(hi_str.c_str(), hi_str.size()); + ASSERT_TRUE(tryReadMyDateTimeText(hi, 6, hi_buf)); + } + + { + // Bounded range: GE+LE merge into a single DateRange only when normalize is enabled. + const auto query = fmt::format( + "select * from default.t_111 where col_datetime >= cast_string_datetime('{}') " + "and col_datetime <= cast_string_datetime('{}')", + lo_str, + hi_str); + + auto rs_operator + = generateRsOperator(table_info_json, query, default_timezone_info, /*enable_trim_minmax*/ false); + EXPECT_EQ(rs_operator->name(), "and"); + auto and_op = std::dynamic_pointer_cast(rs_operator); + ASSERT_NE(and_op, nullptr); + ASSERT_EQ(and_op->getChildren().size(), 2u); + EXPECT_EQ(and_op->getChildren()[0]->name(), "greater_equal"); + EXPECT_EQ(and_op->getChildren()[1]->name(), "less_equal"); + + rs_operator = generateRsOperator(table_info_json, query, default_timezone_info, /*enable_trim_minmax*/ true); + EXPECT_EQ(rs_operator->name(), "date_range"); + EXPECT_EQ( + rs_operator->toDebugString(), + fmt::format( + R"json({{"op":"date_range","col":"col_datetime","class":"EqualityOrInOrBounded","lower":"{}","lower_inclusive":true,"upper":"{}","upper_inclusive":true}})json", + lo, + hi)); + auto reqs = rs_operator->getIndexRequests(); + ASSERT_EQ(reqs.size(), 1u); + EXPECT_EQ(reqs[0].preferred_kind, DM::RSIndexKind::PreferTrim); + ASSERT_TRUE(reqs[0].query_domain.has_value()); + EXPECT_EQ(reqs[0].query_domain->predicate_class, DM::TrimPredicateClass::EqualityOrInOrBounded); + } + + { + // Upper-only bound becomes UpperBounded DateRange when normalize is enabled. + const auto query + = fmt::format("select * from default.t_111 where col_datetime <= cast_string_datetime('{}')", hi_str); + + auto rs_operator + = generateRsOperator(table_info_json, query, default_timezone_info, /*enable_trim_minmax*/ false); + EXPECT_EQ(rs_operator->name(), "less_equal"); + + rs_operator = generateRsOperator(table_info_json, query, default_timezone_info, /*enable_trim_minmax*/ true); + EXPECT_EQ(rs_operator->name(), "date_range"); + EXPECT_EQ( + rs_operator->toDebugString(), + fmt::format( + R"json({{"op":"date_range","col":"col_datetime","class":"UpperBounded","lower":"","lower_inclusive":true,"upper":"{}","upper_inclusive":true}})json", + hi)); + } + + { + // OR must not rewrite children into DateRange, even with normalize enabled. + const auto query = fmt::format( + "select * from default.t_111 where col_datetime >= cast_string_datetime('{}') " + "or col_datetime = cast_string_datetime('{}')", + lo_str, + hi_str); + + auto rs_operator + = generateRsOperator(table_info_json, query, default_timezone_info, /*enable_trim_minmax*/ true); + EXPECT_EQ(rs_operator->name(), "or"); + auto or_op = std::dynamic_pointer_cast(rs_operator); + ASSERT_NE(or_op, nullptr); + ASSERT_EQ(or_op->getChildren().size(), 2u); + EXPECT_EQ(or_op->getChildren()[0]->name(), "greater_equal"); + EXPECT_EQ(or_op->getChildren()[1]->name(), "equal"); + } + + { + // Temporal range AND non-temporal predicate: only the temporal part is rewritten. + const auto query = fmt::format( + "select * from default.t_111 where col_datetime >= cast_string_datetime('{}') " + "and col_datetime <= cast_string_datetime('{}') and col_2 = 666", + lo_str, + hi_str); + + auto rs_operator + = generateRsOperator(table_info_json, query, default_timezone_info, /*enable_trim_minmax*/ true); + EXPECT_EQ(rs_operator->name(), "and"); + auto and_op = std::dynamic_pointer_cast(rs_operator); + ASSERT_NE(and_op, nullptr); + ASSERT_EQ(and_op->getChildren().size(), 2u); + + // Child order follows flatten then rewrite: non-temporal kept first, then DateRange. + bool found_date_range = false; + bool found_equal = false; + for (const auto & child : and_op->getChildren()) + { + if (child->name() == "date_range") + { + found_date_range = true; + EXPECT_EQ( + child->toDebugString(), + fmt::format( + R"json({{"op":"date_range","col":"col_datetime","class":"EqualityOrInOrBounded","lower":"{}","lower_inclusive":true,"upper":"{}","upper_inclusive":true}})json", + lo, + hi)); + } + else if (child->name() == "equal") + { + found_equal = true; + EXPECT_EQ(child->toDebugString(), R"json({"op":"equal","col":"col_2","value":"666"})json"); + } + } + EXPECT_TRUE(found_date_range); + EXPECT_TRUE(found_equal); + } + + { + // Top-level AND with an OR leaf: only the exposed GE/LE bounds are rewritten. + // t >= L AND t <= U AND (t = L OR t = U) => DateRange AND Or(equal, equal) + const auto query = fmt::format( + "select * from default.t_111 where col_datetime >= cast_string_datetime('{}') " + "and col_datetime <= cast_string_datetime('{}') " + "and (col_datetime = cast_string_datetime('{}') or col_datetime = cast_string_datetime('{}'))", + lo_str, + hi_str, + lo_str, + hi_str); + + auto rs_operator + = generateRsOperator(table_info_json, query, default_timezone_info, /*enable_trim_minmax*/ true); + EXPECT_EQ(rs_operator->name(), "and"); + auto and_op = std::dynamic_pointer_cast(rs_operator); + ASSERT_NE(and_op, nullptr); + ASSERT_EQ(and_op->getChildren().size(), 2u); + + bool found_date_range = false; + bool found_or = false; + for (const auto & child : and_op->getChildren()) + { + if (child->name() == "date_range") + { + found_date_range = true; + EXPECT_EQ( + child->toDebugString(), + fmt::format( + R"json({{"op":"date_range","col":"col_datetime","class":"EqualityOrInOrBounded","lower":"{}","lower_inclusive":true,"upper":"{}","upper_inclusive":true}})json", + lo, + hi)); + } + else if (child->name() == "or") + { + found_or = true; + auto or_op = std::dynamic_pointer_cast(child); + ASSERT_NE(or_op, nullptr); + ASSERT_EQ(or_op->getChildren().size(), 2u); + EXPECT_EQ(or_op->getChildren()[0]->name(), "equal"); + EXPECT_EQ(or_op->getChildren()[1]->name(), "equal"); + } + } + EXPECT_TRUE(found_date_range); + EXPECT_TRUE(found_or); + } + + { + // Top-level OR whose child contains AND: do not rewrite the AND inside OR into DateRange. + // (t >= L AND t <= U) OR t = U => still Or(And(...), equal), no date_range + const auto query = fmt::format( + "select * from default.t_111 where (col_datetime >= cast_string_datetime('{}') " + "and col_datetime <= cast_string_datetime('{}')) " + "or col_datetime = cast_string_datetime('{}')", + lo_str, + hi_str, + hi_str); + + auto rs_operator + = generateRsOperator(table_info_json, query, default_timezone_info, /*enable_trim_minmax*/ true); + EXPECT_EQ(rs_operator->name(), "or"); + auto or_op = std::dynamic_pointer_cast(rs_operator); + ASSERT_NE(or_op, nullptr); + ASSERT_EQ(or_op->getChildren().size(), 2u); + + bool found_and = false; + bool found_equal = false; + for (const auto & child : or_op->getChildren()) + { + EXPECT_NE(child->name(), "date_range"); + if (child->name() == "and") + { + found_and = true; + auto and_op = std::dynamic_pointer_cast(child); + ASSERT_NE(and_op, nullptr); + ASSERT_EQ(and_op->getChildren().size(), 2u); + EXPECT_EQ(and_op->getChildren()[0]->name(), "greater_equal"); + EXPECT_EQ(and_op->getChildren()[1]->name(), "less_equal"); + } + else if (child->name() == "equal") + { + found_equal = true; + } + } + EXPECT_TRUE(found_and); + EXPECT_TRUE(found_equal); + } +} +CATCH + +// Design "Must fall back": distinguish shape-level (no PreferTrim) vs eligibility-level +// (PreferTrim DateRange whose domain is outside stored E). +TEST_F(FilterParserTest, TrimMustFallBackShapes) +try +{ + const String table_info_json = R"json({ + "cols":[ + {"comment":"","default":null,"default_bit":null,"id":2,"name":{"L":"col_2","O":"col_2"},"offset":-1,"origin_default":null,"state":0,"type":{"Charset":null,"Collate":null,"Decimal":0,"Elems":null,"Flag":4097,"Flen":0,"Tp":8}}, + {"comment":"","default":null,"default_bit":null,"id":5,"name":{"L":"col_datetime","O":"col_datetime"},"offset":-1,"origin_default":null,"state":0,"type":{"Charset":null,"Collate":null,"Decimal":5,"Elems":null,"Flag":1,"Flen":0,"Tp":12}} + ], + "pk_is_handle":false,"index_info":[],"is_common_handle":false, + "name":{"L":"t_111","O":"t_111"},"partition":null, + "comment":"Mocked.","id":30,"schema_version":-1,"state":0,"tiflash_replica":{"Count":0},"update_timestamp":1636471547239654 +})json"; + + const String in_e = "2020-01-01 00:00:00.00000"; + const String out_high = "2200-01-01 00:00:00.00000"; + const String out_low = "1800-01-01 00:00:00.00000"; + const String hi_in_e = "2020-01-02 00:00:00.00000"; + + auto has_prefer_trim = [](const DM::RSOperatorPtr & op) { + for (const auto & req : op->getIndexRequests()) + { + if (req.preferred_kind == DM::RSIndexKind::PreferTrim) + return true; + } + return false; + }; + + auto expect_no_prefer_trim = [&](const DM::RSOperatorPtr & op, const char * label) { + ASSERT_NE(op, nullptr) << label; + EXPECT_FALSE(has_prefer_trim(op)) << label << " tree=" << op->toDebugString(); + }; + + auto expect_prefer_trim_ineligible = [&](const DM::RSOperatorPtr & op, const char * label) { + ASSERT_NE(op, nullptr) << label; + EXPECT_EQ(op->name(), "date_range") << label; + ASSERT_TRUE(has_prefer_trim(op)) << label; + auto reqs = op->getIndexRequests(); + ASSERT_EQ(reqs.size(), 1u) << label; + ASSERT_TRUE(reqs[0].query_domain.has_value()) << label; + auto dt = std::make_shared(0); + const UInt64 e_lo = DM::TrimMinMax::defaultLowerBoundPacked(*dt); + const UInt64 e_hi = DM::TrimMinMax::defaultUpperBoundPacked(*dt); + EXPECT_FALSE(reqs[0].query_domain->isTrimEligible(e_lo, e_hi)) << label; + }; + + // Shape: col != ... + { + const auto query + = fmt::format("select * from default.t_111 where col_datetime != cast_string_datetime('{}')", in_e); + auto op = generateRsOperator(table_info_json, query, default_timezone_info, /*enable_trim_minmax*/ true); + EXPECT_EQ(op->name(), "not_equal"); + expect_no_prefer_trim(op, "col != literal"); + } + + // Shape: NOT (BETWEEN ...) <=> NOT (GE AND LE) + // Tipb may keep Not(...) or rewrite; either way must not PreferTrim / emit DateRange. + { + const auto query = fmt::format( + "select * from default.t_111 where not (col_datetime >= cast_string_datetime('{}') " + "and col_datetime <= cast_string_datetime('{}'))", + in_e, + hi_in_e); + auto op = generateRsOperator(table_info_json, query, default_timezone_info, /*enable_trim_minmax*/ true); + expect_no_prefer_trim(op, "NOT (BETWEEN)"); + EXPECT_EQ(op->toDebugString().find("date_range"), String::npos) << op->toDebugString(); + } + + // Shape: BETWEEN OR status = 1 (non-temporal Equal stays Normal) + { + const auto query = fmt::format( + "select * from default.t_111 where (col_datetime >= cast_string_datetime('{}') " + "and col_datetime <= cast_string_datetime('{}')) or col_2 = 1", + in_e, + hi_in_e); + auto op = generateRsOperator(table_info_json, query, default_timezone_info, /*enable_trim_minmax*/ true); + EXPECT_EQ(op->name(), "or"); + expect_no_prefer_trim(op, "BETWEEN OR status=1"); + EXPECT_EQ(op->toDebugString().find("date_range"), String::npos); + } + + // Shape: BETWEEN OR col IS NULL + { + const auto query = fmt::format( + "select * from default.t_111 where (col_datetime >= cast_string_datetime('{}') " + "and col_datetime <= cast_string_datetime('{}')) or col_datetime is null", + in_e, + hi_in_e); + auto op = generateRsOperator(table_info_json, query, default_timezone_info, /*enable_trim_minmax*/ true); + EXPECT_EQ(op->name(), "or"); + expect_no_prefer_trim(op, "BETWEEN OR IS NULL"); + EXPECT_EQ(op->toDebugString().find("date_range"), String::npos); + } + + // Shape: non-ColumnRef predicate (CAST stand-in). MockExecutor cannot compile `cast`, + // so use bitand(...) which FilterParser likewise turns into Unsupported / no PreferTrim. + { + auto op = generateRsOperator( + table_info_json, + "select * from default.t_111 where bitand(col_2, 1) > 100", + default_timezone_info, + /*enable_trim_minmax*/ true); + EXPECT_EQ(op->name(), "unsupported") << op->toDebugString(); + expect_no_prefer_trim(op, "function(col) compare literal (CAST stand-in)"); + } + + // Eligibility: col >= 2200 may PreferTrim, but domain is outside default E. + { + const auto query + = fmt::format("select * from default.t_111 where col_datetime >= cast_string_datetime('{}')", out_high); + auto op = generateRsOperator(table_info_json, query, default_timezone_info, /*enable_trim_minmax*/ true); + expect_prefer_trim_ineligible(op, "col >= 2200"); + } + + // Eligibility: col <= 1800 + { + const auto query + = fmt::format("select * from default.t_111 where col_datetime <= cast_string_datetime('{}')", out_low); + auto op = generateRsOperator(table_info_json, query, default_timezone_info, /*enable_trim_minmax*/ true); + expect_prefer_trim_ineligible(op, "col <= 1800"); + } +} +CATCH + // Test cases for unsupported column type TEST_F(FilterParserTest, UnsupportedColumnType) try @@ -763,6 +1256,73 @@ try } CATCH +// ColumnIDToAttrMap may hold empty type when column id is absent from defines. +// parseTiCompareExpr must turn that into Unsupported instead of Equal/In. +TEST_F(FilterParserTest, MissingAttrTypeBecomesUnsupported) +try +{ + const String table_info_json = R"json({ + "cols":[ + {"comment":"","default":null,"default_bit":null,"id":2,"name":{"L":"col_2","O":"col_2"},"offset":-1,"origin_default":null,"state":0,"type":{"Charset":null,"Collate":null,"Decimal":0,"Elems":null,"Flag":4097,"Flen":0,"Tp":8}} + ], + "pk_is_handle":false,"index_info":[],"is_common_handle":false, + "name":{"L":"t_111","O":"t_111"},"partition":null, + "comment":"Mocked.","id":30,"schema_version":-1,"state":0,"tiflash_replica":{"Count":0},"update_timestamp":1636471547239654 +})json"; + + const TiDB::TableInfo table_info(table_info_json, NullspaceID); + QueryTasks query_tasks; + std::tie(query_tasks, std::ignore) = compileQuery( + *ctx, + "select * from default.t_111 where col_2 = 666", + [&](const String &, const String &) { return table_info; }, + getDAGProperties("")); + auto & dag_request = *query_tasks[0].dag_request; + DAGContext dag_context(dag_request, {}, NullspaceID, "", DAGRequestKind::Cop, "", 0, "", log); + ctx->setDAGContext(&dag_context); + + google::protobuf::RepeatedPtrField conditions; + traverseExecutors(&dag_request, [&](const tipb::Executor & executor) { + if (executor.has_selection()) + { + conditions = executor.selection().conditions(); + return false; + } + return true; + }); + + const auto ann_query_info = tipb::ANNQueryInfo{}; + const auto fts_query_info = tipb::FTSQueryInfo{}; + const auto runtime_filter_ids = std::vector(); + const google::protobuf::RepeatedPtrField pushed_down_filters{}; + const google::protobuf::RepeatedPtrField empty_used_indexes{}; + std::unique_ptr dag_query = std::make_unique( + conditions, + ann_query_info, + fts_query_info, + pushed_down_filters, + empty_used_indexes, + table_info.columns, + runtime_filter_ids, + 0, + default_timezone_info); + + // Simulate missing column defines: Attr.type is empty for the referenced column. + DM::FilterParser::ColumnIDToAttrMap column_id_to_attr; + column_id_to_attr[2] = DM::Attr{.col_name = "", .col_id = 2, .type = DataTypePtr{}}; + + auto rs_operator = DM::FilterParser::parseDAGQuery( + *dag_query, + table_info.columns, + column_id_to_attr, + log, + /*enable_trim_minmax*/ false); + EXPECT_EQ(rs_operator->name(), "unsupported"); + // getIndexRequests is called by DMFilePackFilter regardless of trim settings. + EXPECT_NO_THROW(std::ignore = rs_operator->getIndexRequests()); +} +CATCH + // Test cases for not satisfy `column` `op` `literal` TEST_F(FilterParserTest, ComplicatedFilters) try diff --git a/docs/design/2026-07-14-trim-minmax-for-date-types.md b/docs/design/2026-07-14-trim-minmax-for-date-types.md new file mode 100644 index 00000000000..39144d9409d --- /dev/null +++ b/docs/design/2026-07-14-trim-minmax-for-date-types.md @@ -0,0 +1,982 @@ +# Trim Min-Max Index Design for DATETIME, TIMESTAMP, and DATE Types + +- Status: Draft +- Date: 2026-07-14 + +## Summary + +This document proposes an optional pack-level `trim_minmax` index for `DATETIME`, `TIMESTAMP`, and `DATE` columns in TiFlash DeltaMerge storage. The index collects statistics only for non-NULL, non-deleted values within a predefined "effective date range." It prevents a small number of extreme sentinel timestamps from polluting the ordinary min-max index. For example, when an application uses `2100-01-01 00:00:00` to mean "unsettled," the presence of this value in a pack of 8,192 rows raises the ordinary min-max `max` to 2100, weakening the Rough Set Filter (RS Filter) for recent, narrow time-range queries. + +The first version adopts the following key decisions: + +1. The effective date range is the half-open interval `[1900-01-01 00:00:00, 2099-12-01 00:00:00)`. +2. `trim_minmax` stores the min/max of values inside this interval. The ordinary min-max remains unchanged and continues to serve queries that do not satisfy the trim eligibility conditions. +3. The actual bounds, format version, and pack count used by each trim index are persisted directly through `ColumnStat.trim_minmax_index: TrimMinMaxIndexProps`. The `MergedSubFileInfo` associated with the deterministic file name is the sole source of the file location and size. Per-pack low/high flags are merged into the trim index's existing `has_null_marks` byte array, whose on-disk semantics are generalized as `pack_marks`. +4. A reader may select the trim index only according to the actual interval stored in the DMFile. It must not interpret a historical index using the current version's global default interval. +5. A trim index may replace the ordinary min-max for a column only when the reader can prove that out-of-range low and high values each have uniform matching behavior for the predicate. The first version supports equality, IN, and bounded ranges whose match sets are within the effective interval, as well as one-sided ranges whose finite bound is within the effective interval. +6. The reader adjusts the trim rough-check result according to `has_trimmed_low` and `has_trimmed_high`: the presence of a matching trimmed value invalidates `None`, while the presence of a non-matching trimmed value invalidates `All`. +7. The implementation reuses the `MinMaxIndex` payload, serializer, and raw rough check. It adjusts the result through a composition-based trim wrapper instead of making `TrimMinMaxIndex` inherit from `MinMaxIndex`. +8. The first version writes trim indexes only for DMFile V3 / MetaV2. Old DMFiles and unsupported predicates always fall back to the ordinary min-max. + +This design does not change SQL semantics, require DDL, or rewrite historical DMFiles. New and old DMFiles can coexist, with each file independently selecting the trim or ordinary min-max within the same query. + +## Background + +### Current Ordinary Min-Max Generation and Use + +`DMFileWriter` currently generates ordinary min-max indexes for handles, integers, and date/time types: + +```text +DMFileWriter::write + -> DMFileWriter::writeColumn + -> MinMaxIndex::addPack + -> calculate the column's min/max in the current pack +``` + +Relevant implementations: + +- `dbms/src/Storages/DeltaMerge/File/DMFileWriter.cpp` +- `dbms/src/Storages/DeltaMerge/Index/MinMaxIndex.cpp` + +`MinMaxIndex::addPack` ignores values associated with delete marks. Since v6.4.0, it has also excluded NULL from min/max calculation while independently storing `has_null_marks` and `has_value_marks`. The ordinary index for each pack logically contains: + +```text +has_null +has_value +min +max +``` + +On the query side, `FilterParser::parseDAGQuery` converts TiDB DAG predicates into `RSOperator` objects. `DMFilePackFilter` loads min-max indexes for the columns referenced by the predicates and invokes `roughCheck` to produce an `RSResult` for each pack: + +```text +None -> do not read the pack +Some -> read the pack and execute row-level filtering +All -> read the pack, but row-level filtering may be skipped +*Null -> preserve the corresponding NULL semantics +``` + +Relevant implementations: + +- `dbms/src/Storages/DeltaMerge/FilterParser/FilterParser.cpp` +- `dbms/src/Storages/DeltaMerge/Filter/RSOperator.h` +- `dbms/src/Storages/DeltaMerge/File/DMFilePackFilter.cpp` +- `dbms/src/Storages/DeltaMerge/Index/RSResult.h` +- `dbms/src/DataStreams/FilterTransformAction.cpp` + +When the RS result for a block is `All`, `FilterTransformAction` directly constructs an all-true filter and does not execute the actual expression. Therefore, every new index must strictly guarantee that `All` means all visible rows in the pack satisfy the predicate. It cannot merely mean that all values retained by the index satisfy the predicate. + +### Problem Scenario + +Consider a `settle_time DATETIME` column: + +- Normal values are within the past 90 days. +- One out of every 10,000 rows uses `2100-01-01 00:00:00` to represent an unsettled record. +- A stable pack contains approximately 8,192 rows by default. +- Sentinel values are uniformly distributed throughout the table. + +The probability that a pack contains at least one sentinel value is: + +```text +1 - (1 - 1/10000)^8192 β‰ˆ 55.92% +``` + +Therefore, approximately 55.92% of packs will have: + +```text +min = a normal historical timestamp +max = 2100-01-01 00:00:00 +``` + +For the following query: + +```sql +settle_time >= L AND settle_time <= U +``` + +The ordinary min-max can exclude a pack only when either of the following is true: + +```text +pack.max < L +pack.min > U +``` + +The 2100 sentinel breaks the first proof. For a narrow query near the newest end of the time range, many historical packs that are entirely earlier than `L` are incorrectly retained as `Some`. If normal timestamps have good locality within packs, a query for the latest three hours out of the most recent 90 days may increase the theoretical pack read ratio, considering the time predicate alone, from about `1/720 = 0.139%` to about 55.98%. + +If normal timestamps are themselves distributed completely at random within each pack, the ordinary min-max already has poor filtering power, so the marginal benefit of the trim index will also be smaller. Rollout validation must therefore cover data with both good and poor temporal locality. + +### Key Constraints + +1. A specific application sentinel must not be hard-coded into the generic min-max index; doing so could incorrectly prune queries for that value. +2. Readers must support historical DMFiles. Rollout cannot depend on a one-time rewrite of all stable data. +3. During mixed-version operation or rollback, old readers must be able to ignore the new index and continue using the ordinary min-max. +4. `TIMESTAMP` literals are currently converted to UTC by `FilterParser` according to the request time zone. `DATETIME` and `DATE` do not have the same time-zone semantics. +5. The trim index must use the same pack boundaries, NULL rules, and delete-mark rules as the ordinary min-max. +6. `RSResult::All` has the execution semantics of skipping row-level filtering and must not be treated as an ordinary statistical label. + +## Terminology + +| Term | Definition | +| --- | --- | +| ordinary min-max | The complete min/max of non-NULL, non-deleted values currently stored for each DMFile pack | +| effective date range `E` | The persisted half-open interval `[lower, upper)` used when building a particular trim index | +| trim value | A value within `E` that participates in trim min/max calculation | +| trimmed value | A non-NULL, non-deleted value outside `E` that does not participate in trim min/max calculation | +| `pack_marks` | The per-pack `UInt8` array corresponding to the original `has_null_marks` in a min-max index; bit 0 represents NULL, and a trim index additionally uses bits 1 and 2 | +| `has_trimmed_low` | `pack_marks & 0x02`; indicates that a non-NULL, non-deleted value less than `lower_bound` exists | +| `has_trimmed_high` | `pack_marks & 0x04`; indicates that a non-NULL, non-deleted value greater than or equal to `upper_bound` exists | +| query domain `Q` | The set of all non-NULL temporal values that a column predicate may match | +| trim-eligible | The low and high trimmed values have each been proven to have uniform matching semantics for the predicate, so the flags can safely adjust the trim `RSResult` | + +## Goals + +1. Restore pack-level filtering for date/time range queries when sparse extreme dates are uniformly distributed. +2. Produce correct `None`, `Some`, and `All` results for trim-eligible predicates without changing query results. +3. Allow different generations of DMFiles to use different effective date ranges and let readers safely select an index per file. +4. Fall back to the ordinary min-max for old DMFiles, unknown index versions, and missing or corrupted trim metadata. +5. Avoid affecting the existing query hot path when disabled. When enabled, add only bounded write, metadata, and cache overhead for relevant date/time columns. +6. Provide sufficient metrics to validate index use, fallback reasons, pack-pruning gains, and write costs. + +## Non-Goals + +- Changing TiDB SQL semantics or temporal type semantics. +- Requiring users to change DDL or explicitly create an index. +- Supporting `TIME`, durations, strings, or numeric columns in the first version. +- Writing trim indexes for DMFile V1/V2 in the first version. +- Actively backfilling historical DMFiles in the first version. Historical files are naturally rewritten through subsequent merge delta, split, compact, or GC operations. +- Supporting trim predicate analysis for `OR`, `NOT`, `!=`, `NOT IN`, expression columns, or expressions containing casts in the first version. +- Exposing the effective date range as a user-level table or DDL property. + +## Correctness Foundation + +### Pack-Level Trim Semantics + +Let `D` be the set of all values in a pack that participate in the ordinary min-max, and let `E` be the effective date range. The set represented by the trim index is: + +```text +D_trim = D ∩ E +``` + +Let `Q` be the non-NULL match set of the column predicate. When: + +```text +Q βŠ† E +``` + +then: + +```text +D ∩ Q = (D ∩ E) ∩ Q = D_trim ∩ Q +``` + +Therefore: + +- When the trim min-max proves that `D_trim ∩ Q` is empty, it may safely return `None`. +- The trim min-max cannot return a final `All` merely because `D_trim βŠ† Q`, because `D - E` may still contain non-matching values. +- For equality, IN, and bounded ranges, low and high trimmed values are both non-matching. Therefore, when a trimmed value exists in either direction, an `All` returned by the trim index must be downgraded to `Some`. +- NULL still participates in `RSResult` composition through the trim index's `has_null` mark and is not a trimmed value. + +Directional flags also make it safe to handle one-sided ranges whose bound is within `E`: + +| Predicate type | Low trimmed value | High trimmed value | +| --- | --- | --- | +| `col >= T` / `col > T` | Never matches | Always matches | +| `col <= T` / `col < T` | Always matches | Never matches | + +Consequently, if the raw trim result is `None` but a trimmed value that must match exists, the result must be downgraded to `Some`. If the raw trim result is `All` but a trimmed value that must not match exists, it must also be downgraded to `Some`. Here, "downgrade" means giving up pack-level certainty and retaining row-level filtering. It does not mean upgrading `None` to `All`. + +The following counterexample shows why checking only whether the SQL literal is inside the effective interval is insufficient: + +```text +E = [1900, 2099-12) +pack = {2100-01-01} +predicate = col >= 2020-01-01 +``` + +The literal 2020 is inside `E`, but the query domain is `[2020, +∞)` and includes 2100. Using an empty trim min-max to return `None` would incorrectly omit a matching row. This design uses `has_trimmed_high` to identify a high value that must match and adjusts `None` to `Some`, safely supporting this one-sided comparison. + +Another counterexample demonstrates why low/high pack marks are necessary: + +```text +E = [1900, 2099-12) +pack = {2021-01-01, 2100-01-01} +predicate = col BETWEEN 2020-01-01 AND 2022-01-01 +``` + +After trimming, `min=max=2021-01-01`, but the 2100 value in the pack does not match. The trim rough check must return `Some` rather than `All`. + +### AND + OR Composition Correctness + +Rough Set composition for `And` / `Or` is independent of which physical index each leaf uses. Correctness does **not** require the whole tree to select trim or ordinary uniformly. It requires each leaf's `roughCheck` to be individually sound and conservative; `And` then combines results with `&&` and `Or` with `||`. + +#### Per-predicate index selection + +Trim indexes may be cached per column in `RSCheckParam.trim_indexes`, but a leaf must not use that trim index unless **its own** `DateQueryDomain` is trim-eligible for the stored `E`. This check is enforced at use time by `getTrimRSIndex(param, attr, query_domain)`: + +1. Look up the column's loaded trim index (if any). +2. Call `query_domain.isTrimEligible(stored_lower, stored_upper)`. +3. If ineligible, return no trim index so the leaf falls back to ordinary min-max (when loaded) or `Some`. + +Therefore two leaves on the same column may legally use different indexes in one query: + +```text +DateRange(t, [L, U]) AND Or(t = A, t = B) +``` + +| Leaf | Index choice | +| --- | --- | +| `DateRange` | Trim iff `[L, U]` is eligible for stored `E` | +| `Equal(A)` | Trim iff `A ∈ E` | +| `Equal(B)` | Trim iff `B ∈ E` (independent of `A`) | + +A dangerous anti-pattern, and the reason eligibility must be per predicate rather than per column, is: + +```text +E = [1900, 2099-12) +pack = {2200-01-01} +predicate = (t = 2020-01-01) OR (t = 2200-01-01) +``` + +If both equals shared a column-level trim selection without re-checking eligibility, `t = 2200` would consult an empty `D_trim` and could return `None`, causing the whole `Or` to drop a matching pack. With per-predicate eligibility, `t = 2200` must use the ordinary min-max and keep the pack. + +#### Normalize interaction with AND / OR + +`normalizeTemporalRangesForTrim` only flattens **top-level** `And` nodes and merges exposed temporal `GE` / `GT` / `LE` / `LT` leaves into `DateRange`. It does not rewrite children under `Or` / `Not`. + +| Shape | Rewrite behavior | Correctness implication | +| --- | --- | --- | +| `t >= L AND t <= U` | Becomes `DateRange` PreferTrim | Covered by pack-level trim semantics above | +| `t >= L OR t = X` | Unchanged `Or` | No incorrect DateRange merge | +| `t >= L AND t <= U AND (t = A OR t = B)` | Outer bounds β†’ `DateRange`; `Or` kept as a leaf | `DateRange` and each `Equal` still select indexes independently | +| `(t >= L AND t <= U) OR t = X` | Unchanged top-level `Or` | Inner `And` is not rewritten; `Greater`/`Less` leaves continue to use ordinary min-max (they do not request PreferTrim) | + +Not rewriting under `Or` is an intentional conservative choice: it may forgo trim benefit, but it must not invent a single range whose match set is not equivalent to the original disjunction. + +#### Soundness checklist for mixed trees + +For any `And` / `Or` tree that mixes trim-capable and ordinary leaves: + +1. Each PreferTrim leaf must re-validate eligibility against stored `E` at `roughCheck` time, not only at load time. +2. Every trim-path result must still apply pack-mark correction (`None`/`All` downgrades for matching / non-matching trimmed values). +3. `All` remains the only RSResult that may skip row-level filtering, so a trim-derived `All` must remain sound after correction. +4. If any leaf cannot prove a safe trim result, it must degrade to ordinary min-max or `Some`; logical composition then stays conservative. + +Under these rules, shapes such as `DateRange AND Or(Equal, Equal)` do not introduce an additional correctness hazard beyond the single-predicate trim foundation: composition preserves leaf soundness, and index choice is decided per leaf rather than per column or per logical operator. + +## Design + +### Overall Architecture + +```text +DMFile write path + -> ordinary MinMaxIndex: all non-NULL, non-deleted values + -> TrimMinMaxIndex: non-NULL, non-deleted values within E + -> ColumnStat.trim_minmax_index: + - TrimMinMaxIndexProps: format version + encoded bounds + pack count + -> MergedSubFileInfo[.trim.idx]: + - merged file number + offset + size + -> trim index pack_marks[pack_count]: + - bit 0: has_null + - bit 1: has_trimmed_low + - bit 2: has_trimmed_high + +Query DAG + -> FilterParser + - build the existing RSOperator + - normalize the predicate type and temporal bounds + -> DMFilePackFilter (per DMFile) + - predicate bounds are within stored E and low/high semantics are provable: select trim + - otherwise: select ordinary min-max + -> roughCheck + - matching trimmed value + None: Some + - non-matching trimmed value + All: Some + - other None/All: unchanged + - Some: Some +``` + +### Effective Date Range + +The default interval for the first version is: + +```text +[1900-01-01 00:00:00, 2099-12-01 00:00:00) +``` + +A half-open interval is used so fractional seconds in `DATETIME(1..6)` remain unambiguous, for example `2099-11-30 23:59:59.999999` is inside `E` while `2099-12-01 00:00:00` is not. The exclusive upper bound is `2099-12-01` rather than `2100-01-01` to leave a buffer before the common `2100-01-01` sentinel: `TIMESTAMP` literals converted across time zones or DST must not accidentally move the exclusive edge onto an unexpected calendar day and weaken outlier classification around that sentinel. + +The bounds are persisted using the column's internal encoding rather than as time strings: + +- `DATE` uses the packed value of `DataTypeMyDate`. +- `DATETIME` and `TIMESTAMP` use the packed value of `DataTypeMyDateTime`. +- The field type of both bounds must match the nested type of the indexed column. +- The format contract for `format_version = 1` always interprets the bounds as `[lower, upper)`; the metadata does not separately store boundary semantics. + +`TIMESTAMP` query literals continue to use `FilterParser::convertFieldWithTimezone` and are converted into UTC packed values before Rough Set analysis. `DATETIME` and `DATE` are compared as calendar values; "UTC+0" is not part of their type semantics. + +### TrimMinMaxIndex Data Model + +The trim min-max reuses the core representation of the ordinary `MinMaxIndex`: + +```text +pack_marks[pack_count] // same physical position as the existing has_null_marks +has_value_marks[pack_count] +minmaxes[pack_count * 2] +``` + +The V1 bit layout of `pack_marks` is: + +| Bit | Mask | Ordinary min-max | Trim min-max | +| --- | --- | --- | --- | +| 0 | `0x01` | `has_null` | `has_null` | +| 1 | `0x02` | Must be 0 | `has_trimmed_low` | +| 2 | `0x04` | Must be 0 | `has_trimmed_high` | +| 3..7 | `0xf8` | Must be 0 | Must be 0; reserved | + +The remaining fields have the following semantics: + +- `has_value_marks` indicates whether the pack contains at least one non-NULL, non-deleted value within `E`. +- `minmaxes` is calculated only from values within `E`. +- NULL, low/high flags, and the trim min/max use the same pack order and all reside in the same trim index payload. + +The byte layout and existing content of the ordinary `.idx` remain unchanged: historical mark values are already either `0x00` or `0x01`. After generalizing the internal concept from `has_null_marks` to `pack_marks`, every NULL check in the code must use `(pack_marks[i] & 0x01) != 0` instead of treating the entire byte as a Boolean; otherwise, a low/high bit would be misinterpreted as NULL. Access through `hasNull()`, `hasTrimmedLow()`, and `hasTrimmedHigh()` accessors is recommended. + +`TrimMinMaxIndex` is not a subclass of `MinMaxIndex`. The current `MinMaxIndex` state is private, `read()` always constructs the base class, `checkCmp` is a non-virtual template method, and the query path calls through `MinMaxIndexPtr`. Adding virtual interfaces and a specialized factory for inheritance would not simplify the disk format. The first version uses composition: + +```cpp +struct TrimMinMaxIndex +{ + MinMaxIndexPtr minmax; + + RSResults roughCheck( + TrimPredicateClass predicate_class, + size_t start_pack, + size_t pack_count) const; +}; +``` + +`MinMaxIndex` performs generic serialization and the raw min-max check. The trim wrapper or `DateRange` operator reads the pack-mark accessors and adjusts `None` / `All`. The wrapper stores no additional per-pack arrays. + +The trim index uses a separate subfile name, logically: + +```text +.trim.idx +``` + +In DMFile V3, this small file is written into a merged file just like the ordinary min-max and marks. `MergedSubFileInfo` stores its physical file number, offset, and size. + +Trim data is not appended to the existing `.idx`. `MinMaxIndex::read` currently requires the number of bytes actually consumed to match the ordinary `index_bytes` exactly. Directly extending the existing file would break old readers. + +### `ColumnStat.trim_minmax_index` Metadata + +The first version neither adds a separate MetaV2 block nor appends trim to the existing `ColumnStat.indexes = 104`. Instead, `dmfile.proto` adds an independent optional field to `ColumnStat` whose type is directly `TrimMinMaxIndexProps`: + +```protobuf +message ColumnStat { + // existing fields... + repeated DMFileIndexInfo indexes = 104; + + // Internal DMFile trim min-max; at most one per column. + optional TrimMinMaxIndexProps trim_minmax_index = 105; +} + +message TrimMinMaxIndexProps { + optional uint32 format_version = 1; + + // Internal encoding matching the nested type of the owning ColumnStat. + optional bytes lower_bound = 2; + optional bytes upper_bound = 3; + + optional uint64 pack_count = 4; +} +``` + +The actual field number will be assigned during implementation according to protobuf compatibility rules; the structure above defines the data contract. `TrimMinMaxIndexProps` contains information specific to an internal DMFile pack index. It does not reuse `DMFileIndexInfo`, `IndexFilePropsV2`, `IndexFileKind`, or the DDL local-index lifecycle. + +Field sources and omission rules: + +- The column ID comes from the enclosing `ColumnStat.col_id` and is not stored again. +- The file name is deterministically derived from the column ID as `.trim.idx`. +- The physical merged-file number, offset, and size are stored only in the `MergedSubFileInfo` corresponding to that file name. `TrimMinMaxIndexProps` does not duplicate the file size. +- Trim is an internal index. It does not correspond to a TiDB DDL index, has no kind or index ID, and does not enter local-index APIs that query by index ID. +- Per-pack flags exist only in the trim index payload's `pack_marks`; protobuf does not store a per-pack array. + +Metadata constraints: + +1. `format_version = 1` always uses the half-open interval `[lower_bound, upper_bound)`. Any future change to boundary semantics must increment the version. A reader falls back to the ordinary min-max when it encounters an unknown version. +2. The bounds must be decodable as the owning column's nested type and must satisfy `lower_bound < upper_bound`. +3. `pack_count` must equal the number of packs in the DMFile. +4. The `.trim.idx` file name derived from the owning column must be present in `MergedSubFileInfo`, with a valid number, offset, and size. This size is the sole source of the file size for loading and checksum-frame calculation. +5. The counts of decoded `pack_marks`, `has_value_marks`, and min/max cells must agree and equal `pack_count`. +6. Bits 3 through 7 of the trim index's `pack_marks` must be zero. +7. The `ColumnStat` metadata, `MergedSubFileInfo`, and index payload must be generated and published atomically as part of the same immutable DMFile generation. A reader must not reuse or combine them across DMFiles. +8. Soft fallback to the ordinary min-max (do not use trim) when any of the following holds: trim metadata is absent; the format version is unknown; props fail logical checks such as undecodable bounds, `lower_bound >= upper_bound`, or `pack_count` mismatch; or the deterministic `.trim.idx` entry is simply missing from `MergedSubFileInfo` (including the orphan-subfile case after an old node rewrote ColumnStat and dropped field 105). Soft fallback must not invent a speculative result other than what the ordinary path would produce. +9. Hard failure (throw / existing DMFile corruption policy), not soft fallback, when trim is selected but the on-disk location is structurally impossible or the payload is definitively corrupt. Examples: a present `MergedSubFileInfo` with an invalid merged-file number, or with `offset`/`size` that cannot fit the merged file; checksum mismatch on the trim subfile; decoded pack-mark reserved bits nonzero after load. Invalid `MergedSubFileInfo` geometry must not be rewritten into a silent ordinary-minmax fallback in selection. + +The ColumnStat protobuf is protected by the DMFileMetaV2 checksum, while the trim index subfile uses the existing DMFile checksum mechanism. Logical metadata mismatches soft-fallback; physical corruption of a selected trim subfile or an impossible `MergedSubFileInfo` is surfaced as corruption rather than hidden behind soft fallback. + +The compatibility-critical property is that field 105 does not belong to `indexes = 104`, which old versions iterate over and validate strictly. Old readers treat all of field 105 as an unknown protobuf field. It does not trigger `ColumnStat::integrityCheckIndexInfoV2` and does not require recognition of a new `IndexFileKind`. Old readers continue to read the ordinary min-max. + +This compatibility guarantees safe reading only. After converting protobuf into an old C++ `ColumnStat`, an old version will not preserve field 105. If an old node subsequently rewrites ColumnStat metadata, for example while adding a local index to a DMFile, it may discard the trim metadata. A new reader must then treat `.trim.idx` as unavailable and fall back to the ordinary min-max. The remaining subfile is only a space issue and is reclaimed when the DMFile is later rewritten; it must not affect query correctness. + +No field is added directly to `PackStat` or `PackProperty`. MetaV2 currently serializes these structures using their raw POD layouts, so changing `sizeof` would break the old format. + +### Write Path + +For each supported temporal column, `DMFileWriter::Stream` adds an optional trim `MinMaxIndex` builder. The ordinary and trim min-max values are calculated during the same pack traversal to avoid scanning the column twice. This reuses the builder and serializer and does not extend `MinMaxIndex` through inheritance. + +Pseudocode: + +```cpp +UInt8 trim_pack_mark = 0; + +for (row : pack) +{ + if (isDeleted(row)) + continue; + + if (isNull(row)) + { + ordinary.has_null = true; + trim_pack_mark |= 0x01; // has_null + continue; + } + + ordinary.updateMinMax(row.value); + + if (lower <= row.value && row.value < upper) + trim.updateMinMax(row.value); + else if (row.value < lower) + trim_pack_mark |= 0x02; // has_trimmed_low + else + trim_pack_mark |= 0x04; // has_trimmed_high +} +``` + +After each pack is written: + +1. The ordinary min-max appends one cell through the existing path. +2. The trim min-max appends one cell. If there is no in-range value, `has_value=false`. +3. `trim_pack_mark` is appended to the trim index's `pack_marks`. +4. During finalization, the writer writes the trim subfile, registers its `MergedSubFileInfo`, and stores `TrimMinMaxIndexProps` in the owning `ColumnStat.trim_minmax_index`. + +The implementation adds an internal `appendPack(pack_mark, has_value, min, max)` or equivalent builder API to `MinMaxIndex` for use by both ordinary and trim writers. The ordinary path permits only `pack_mark & ~0x01 == 0`; the trim path permits `0x01 | 0x02 | 0x04`. Deserialization continues to reuse the generic payload reader, after which the trim loader validates the mark mask and pack count. + +No trim index is generated for: + +- Internal columns such as handle, version, and delete mark. +- `TIME` and duration. +- Nested types other than `MyDate` / `MyDateTime`. +- Empty DMFiles. +- Any column when `dt_enable_trim_minmax` is disabled. + +During finalization, the writer may check whether a column has any trimmed value anywhere in the DMFile. If all `pack_marks & 0x06` values are zero, the ordinary and trim min-max indexes are equivalent for non-NULL values. The trim subfile and metadata may then be omitted, avoiding storage overhead for files without abnormal values. The bit-0 NULL mark does not affect this decision. + +### Query-Domain Analysis + +`FilterParser` currently converts each comparison expression independently into operators such as `GreaterEqual` and `LessEqual`. A single literal is insufficient to determine whether the complete query domain belongs to the effective interval, so a temporal column query-domain normalization step is required. + +The first version supports the following trim-eligible forms: + +```sql +time_col = T +time_col IN (T1, T2, ...) +time_col >= L AND time_col <= U +time_col > L AND time_col < U +time_col >= L +time_col > L +time_col <= U +time_col < U +``` + +The rules are: + +- The query domain for equality is the singleton `{T}`. +- All non-NULL values in an IN predicate must be within the stored `E`. +- After type and time-zone conversion, a bounded range's lower and upper bounds must form `Q βŠ† E`. +- The bound of a lower-bounded range must be within the stored `E`. A low trimmed value then never matches, while a high trimmed value always matches. +- The bound of an upper-bounded range must be within the stored `E`. A low trimmed value then always matches, while a high trimmed value never matches. +- The lower and upper bounds may come from the same `LogicalAnd`, or from the top-level AND ultimately formed by `DAGQueryInfo::filters` and `pushed_down_filters`. +- When multiple lower or upper bounds exist for the same column, select the semantically strongest lower and upper bounds. +- In the first version, no trim query domain is generated for a branch involving OR, NOT, NotEqual, NotIn, IsNull, a function, or a cast. + +A new `DateRange` RSOperator should represent an already-normalized date range, including bounded and one-sided ranges. This operator affects only the rough check. The actual row-level filter continues to be built from the original DAG expression; the SQL execution expression is not changed. + +To allow `DMFilePackFilter` to select a normal or trim index per DMFile, extend the index request interface so that an operator can declare: + +```cpp +struct RSIndexRequest +{ + ColId col_id; + RSIndexKind preferred_kind; // Normal or PreferTrim + std::optional query_domain; +}; +``` + +`RSCheckParam` stores normal and trim indexes separately, preventing an overwrite when the same column participates in both a trim-eligible range and another ordinary predicate: + +```cpp +struct RSCheckParam +{ + ColumnIndexes normal_indexes; + TrimColumnIndexes trim_indexes; // values contain composition-based TrimMinMaxIndex wrappers + TrimMinMaxInfos trim_infos; // parsed from ColumnStat.trim_minmax_index +}; +``` + +If the query domain of a temporal operator satisfies the trim conditions for a DMFile, only the trim index is loaded by preference. If another operator on the same column also needs the ordinary min-max, both types may coexist. Index caches use distinct keys containing at least a stable DMFile identity, the column ID, and the index kind. Because a DMFile is an immutable generation, the bounds and the index payload containing pack marks cannot change independently under the same identity. + +### Per-DMFile Index Selection + +For every temporal column request and every DMFile, independently execute: + +```cpp +if (!trim_read_enabled) + choose NORMAL; +else if (!trim_meta_exists(col_id)) + choose NORMAL; +else if (!readerSupports(trim_meta.format_version)) + choose NORMAL; +else if (!metaAndIndexAreConsistent(trim_meta)) + choose NORMAL; +else if (!query_domain.isTrimEligible(trim_meta.range)) + choose NORMAL; +else + choose TRIM; +``` + +`isTrimEligible` is not equivalent to a simple `Q βŠ† E` check. Equality, IN, and bounded ranges still require their complete match sets to be inside `E`. A one-sided range instead requires its finite bound to be within `E` and passes the predicate classification to the rough check so that low and high trimmed values can be identified as "always matching" or "never matching." + +Selection must use `trim_meta.range` rather than the current process's default range. This safely supports: + +```text +DMFile A: E=[1900, 2099-12), use trim +DMFile B: no trim, use normal +DMFile C: future version E=[1800, 2200), decide according to C's metadata +``` + +If the reader does not support the metadata version, or logical selection checks fail (missing meta, missing `.trim.idx` entry, undecodable/invalid bounds, `pack_count` mismatch, ineligible query domain), it soft-falls back to the ordinary min-max rather than returning a speculative result other than `Some`. Soft fallback deliberately does **not** apply to a present but structurally invalid `MergedSubFileInfo` (impossible number/offset/size) or to a definitive trim-subfile checksum / pack-mark corruption: those follow the existing DMFile data-corruption policy and must throw rather than be rewritten as an ordinary-minmax soft fallback. + +### Trim Rough Check and `pack_marks` + +For each pack, the reader decodes: + +```cpp +const UInt8 pack_mark = trim_index.minmax->packMark(pack_id); +const bool has_null = (pack_mark & 0x01) != 0; +const bool has_trimmed_low = (pack_mark & 0x02) != 0; +const bool has_trimmed_high = (pack_mark & 0x04) != 0; +``` + +Based on the predicate class, it determines whether a trimmed value that must match or must not match exists: + +| Predicate type | `trimmed_match_exists` | `trimmed_nonmatch_exists` | +| --- | --- | --- | +| Equality / IN / bounded range with `Q βŠ† E` | false | `has_trimmed_low || has_trimmed_high` | +| Lower-bounded range with bound in `E` | `has_trimmed_high` | `has_trimmed_low` | +| Upper-bounded range with bound in `E` | `has_trimmed_low` | `has_trimmed_high` | + +The trim min-max first calculates a result using the existing RoughCheck rules and then applies these conservative adjustments: + +| Raw trim result | Condition | Final result | +| --- | --- | --- | +| `None` | `trimmed_match_exists=false` | `None` | +| `None` | `trimmed_match_exists=true` | `Some` | +| `NoneNull` | `trimmed_match_exists=false` | `NoneNull` | +| `NoneNull` | `trimmed_match_exists=true` | `SomeNull` | +| `Some` / `SomeNull` | Any | Unchanged | +| `All` | `trimmed_nonmatch_exists=false` | `All` | +| `All` | `trimmed_nonmatch_exists=true` | `Some` | +| `AllNull` | `trimmed_nonmatch_exists=false` | `AllNull` | +| `AllNull` | `trimmed_nonmatch_exists=true` | `SomeNull` | + +These rules only downgrade a certain result to `Some`. The flags never upgrade `None` to `All` or `Some` to a certain result. This safely handles one-sided queries even when a trim pack has no in-range value. For example, for a pack containing only 2100 and `col >= 2020`, the raw trim result may be `None`, but `has_trimmed_high` adjusts it to `Some`. A bounded-range query can still leave this pack as `None` and skip it. + +### NULL, Deletion, and MVCC + +The trim index follows the current ordinary min-max rules: + +- NULL does not participate in min/max. +- A non-deleted NULL sets the trim index's `has_null`. +- NULL does not set `has_trimmed_low` or `has_trimmed_high`. +- Values associated with delete marks participate in neither normal nor trim and do not set the low/high bits in `pack_marks`. +- `has_value=false` is set when a pack contains no non-deleted in-range value. + +This keeps the trim and ordinary min-max indexes consistent in their three-valued RS semantics. If the ordinary min-max definition of the set of MVCC-visible values changes in the future, trim generation must change with it; the two indexes must not represent different row sets. + +### Configuration and Switches + +The first version provides one internal setting: + +```text +dt_enable_trim_minmax +``` + +- When enabled, new DMFiles may generate trim indexes, and readers may select trim (including temporal-range normalize for PreferTrim). +- When disabled, writers skip trim generation, and readers ignore existing trim metadata/subfiles and fall back to the ordinary min-max. +- Default is disabled; enable gradually through canaries. Note that a single switch means write and read are rolled out together. +- The effective interval is not dynamically configurable in the first version, avoiding configuration drift within a process. It is still persisted in metadata to preserve correctness during future format evolution. + +### Observability + +Add query- or instance-aggregated counters and timings: + +```text +trim_minmax_index_load_count +trim_minmax_index_load_bytes +trim_minmax_index_load_time +trim_minmax_selected_packs +trim_minmax_none_packs +trim_minmax_some_packs +trim_minmax_all_packs +trim_minmax_none_downgraded_packs +trim_minmax_all_downgraded_packs +trim_minmax_fallback_count{reason} +trim_minmax_metadata_lost_count +trim_minmax_orphan_subfile_count +trim_minmax_write_bytes +trim_minmax_write_time +``` + +`fallback reason` must distinguish at least: + +```text +disabled +no_meta +unsupported_version +predicate_boundary_outside_range +unsupported_expression +metadata_mismatch +index_missing +``` + +Debug logs record the DMFile, column ID, stored range, query domain, selected index type, and pack-filtering ratio. In the long term, trim pack statistics should be incorporated into `ScanContext` so they can be displayed by `EXPLAIN ANALYZE`. In the first phase, ProfileEvents, instance metrics, and debug logs are sufficient for validation. + +## Compatibility and Invariants + +### Query-Correctness Invariants + +1. A trim index must not make any originally matching row disappear. +2. A trim index must not allow a non-matching trimmed value into the result through `All`. +3. Equality, IN, and bounded ranges may select trim only when `Q βŠ† stored E`. A one-sided range may select trim only when its finite bound is within stored `E` and the low/high matching semantics are known. +4. A reader must not use trim when logical selection checks fail (missing or unsupported meta, missing `.trim.idx` entry, ineligible domain). A present but structurally invalid `MergedSubFileInfo` or a corrupt trim payload is not a soft-fallback case; it follows the DMFile corruption policy. +5. The row-level filter is always retained and may be skipped only when the final RSResult is a strictly correct `All`. +6. Boundary semantics for `format_version = 1` are always `[lower_bound, upper_bound)` and must not be reinterpreted by runtime configuration. +7. NULL semantics read only `pack_marks & 0x01`. Low/high bits must not affect NULL checks for the ordinary min-max. + +### Disk-Format Compatibility + +- The ordinary `.idx` format is unchanged, and old readers continue to read the ordinary min-max. +- Pack marks in an ordinary `.idx` remain restricted to `0x00` / `0x01`. Extended trim bits appear only in the separate `.trim.idx`. +- Trim uses a separate subfile, so an old reader does not interpret extra bytes as part of the ordinary index. +- Trim metadata uses independent optional field 105 in `ColumnStat` and is not added to `indexes = 104`, which old readers validate strictly. +- Old readers ignore field 105. New readers fall back to normal when reading old DMFiles that lack the field. +- `format_version` versions both the serialized structure and boundary semantics. Unsupported versions fall back to normal. +- The first version does not modify the raw `PackStat` / `PackProperty` layout. +- The first version writes only V3 / MetaV2, avoiding simultaneous extensions to the V1/V2 metadata formats. + +### Rolling Upgrade and Downgrade + +Recommended sequence: + +1. First deploy new binaries capable of parsing `ColumnStat.trim_minmax_index`, with `dt_enable_trim_minmax` disabled (default). +2. Enable `dt_enable_trim_minmax` on a small canary so those nodes both generate trim on new DMFiles and select trim when eligible. +3. Verify that old readers ignore field 105 and read the ordinary min-max. Also verify that, if an old node rewrites metadata and loses trim information, a new reader safely falls back. +4. Expand the rollout. + +For downgrade, disable `dt_enable_trim_minmax` first. Existing trim subfiles and field 105 do not affect the ordinary min-max. Compatibility tests must verify both "new write, old read" and "old-version metadata rewrite." If the target old version cannot safely ignore field 105 or an additional merged subfile, the switch must remain disabled during mixed-version operation. Losing trim metadata during an old-version rewrite is an allowed safe degradation, but it must be recorded in metrics and the corresponding loss of trim benefit must be accepted for that DMFile. + +## Performance and Resource Overhead + +### Index Space + +For `MyDateTime`, the trim min-max for each pack contains approximately: + +```text +min + max 16 bytes +pack_marks 1 byte +has_value_marks 1 byte +``` + +With the current byte-array representation of `MinMaxIndex`, the uncompressed size is approximately 18 bytes per pack per column. With 8,192 rows per pack: + +```text +18 / 8192 β‰ˆ 0.0022 bytes/row/column +``` + +The low/high flags reuse the trim min-max's existing per-pack `has_null_marks` byte. Therefore, they add no bytes relative to a trim min-max payload and do not make MetaV2 grow with the number of packs. For a DMFile with about one million rows, 123 packs, and five temporal columns, this saves approximately `123 Γ— 5 = 615` bytes of MetaV2 content compared with storing the flags separately in metadata. + +### Write CPU + +If min/max is scanned twice independently, index-building CPU for temporal columns may nearly double. The design requires normal and trim to be updated during one traversal. The common path adds only two bound comparisons and one conditional branch. + +### Reads and Cache + +- If a query is trim-eligible and the column has no other normal-index request, load only trim and not the ordinary min-max. +- A fallback query loads only the ordinary index. +- If the same column has both a trim-eligible predicate and another ordinary predicate, both indexes may be loaded. +- Trim and ordinary use distinct cache keys. Pack marks are already included in the first byte array accounted for by `MinMaxIndex::byteSize()` and do not require separate cache weight. +- A DMFile with no trimmed value does not persist trim, avoiding overhead with no benefit. + +## Phased Implementation and Rollout + +### Phase A: Format and Compatibility + +- Add `TrimMinMaxIndexProps` and `ColumnStat.trim_minmax_index = 105` and implement trim-specific parsing and validation. Do not modify `DMFileIndexInfo`, `IndexFilePropsV2`, or `IndexFileKind`. +- Add trim subfile naming, merged-file location, and cache keys. +- Consolidate internal `has_null_marks` reads behind pack-mark accessors and verify that ordinary min-max interprets only bit 0. +- Ensure new readers can read old files and fall back to normal for every trim anomaly. +- Keep read/write switches disabled by default. +- Complete tests for new-write/old-read, old-write/new-read, old-version metadata rewrite, CN/disaggregated file paths, and checksums. + +### Phase B: Index Writing + +- Generate normal, trim, and trim `pack_marks` in one traversal in `DMFileWriter`. +- Generate them only for V3 user columns of `MyDate` / `MyDateTime` types. +- Canary writes and observe write throughput, CPU, DMFile size, and metadata size. + +### Phase C: Query Domain and Read Path + +- Add top-level AND temporal-range normalization and the `DateRange` operator. +- Use a composition-based trim wrapper to invoke the generic `MinMaxIndex` without introducing a virtual base interface or derived-class deserialization. +- Support trim eligibility for equality, IN, bounded ranges, and one-sided ranges. +- Select indexes according to each DMFile's stored range. +- Use low/high flags to conservatively implement `None -> Some`, `NoneNull -> SomeNull`, `All -> Some`, and `AllNull -> SomeNull` adjustments. +- Canary reads and compare query results, scanned rows, and pack-filtering ratios. + +### Phase D: Default Enablement and Natural Migration + +- Gradually enable reads and writes by default after compatibility and performance thresholds are met. +- Continue using normal for old DMFiles without a full backfill. +- Naturally increase trim coverage through existing merge delta, split, compact, and GC operations. +- Retain the read/write kill switch for at least one full release cycle. + +## Validation Strategy + +### Unit Tests + +#### TrimMinMaxIndex + +Cover at least these packs: + +```text +all values within E +all values below E +all values above E +both low and high outliers +normal value + 2100 sentinel +NULL only +NULL + normal value + outlier +delete mark + normal value + outlier +no valid values +``` + +Verify min/max, `has_value`, pack-mark accessors, and serialization round trips. Cover at least `0x00`, `0x01` (NULL), `0x02` (low), `0x04` (high), `0x06` (low + high), and `0x07` (NULL + low + high). A trim V1 reader must reject an index payload with nonzero bits 3 through 7. Historical ordinary `.idx` values `0x00` / `0x01` must remain compatible. + +#### RSResult + +Verify in particular: + +```text +pack={2021, 2100}, query=[2020, 2022] -> Some; must not be All +pack={2100}, query=[2020, 2022] -> None +pack={2021}, query=[2020, 2022] -> All +pack={NULL, 2021}, query=[2020, 2022] -> AllNull or an equivalent result requiring row-level filtering +pack={2100}, query>=2020 -> Some; must not be None +pack={1800}, query>=2020 -> None +pack={1800}, query<=2020 -> Some; must not be None +pack={2100}, query<=2020 -> None +pack={1800, 2100}, query>=2020 -> Some +``` + +#### Query-Domain Analysis + +Trim may be used for: + +```sql +col = '2020-01-01' +col IN ('2020-01-01', '2021-01-01') +col >= '2020-01-01' AND col <= '2020-01-02' +col >= '2020-01-01' +col < '2020-01-01' +``` + +Must fall back: + +```sql +col >= '2200-01-01' +col <= '1800-01-01' +col != '2020-01-01' +NOT (col BETWEEN ...) +col BETWEEN ... OR status = 1 +col BETWEEN ... OR col IS NULL +CAST(col AS ...) = ... +``` + +### Temporal-Type Tests + +- `DATE`, `DATETIME(0)`, `DATETIME(3)`, and `DATETIME(6)`. +- `TIMESTAMP` in UTC, fixed-offset, and named time zones, including DST boundaries. +- The lower and upper bounds themselves. +- `2099-11-30 23:59:59.999999` (inside `E`). +- `2099-12-01 00:00:00` and `2100-01-01 00:00:00` (outside `E`). +- Zero dates, invalid-date compatibility values, and other out-of-range packed values. +- Nullable and NotNull. + +### DMFile Format and Compatibility Tests + +- Old DMFile -> new reader. +- New DMFile -> old reader within the supported compatibility range. +- Missing `ColumnStat.trim_minmax_index`, unknown version, incorrect `pack_count`, undecodable bounds, or `lower_bound >= upper_bound` soft-falls back to ordinary. +- Missing `MergedSubFileInfo` for the deterministic `.trim.idx` file name soft-falls back to ordinary (including orphan residual after meta loss). +- Present `MergedSubFileInfo` with invalid number/offset/size must hard-fail under the DMFile corruption policy; it must not soft-fallback in selection. +- Field 105 must not appear in `indexes = 104`, and an old reader must not enter `integrityCheckIndexInfoV2` for it. +- An old reader rewrites ColumnStat and drops field 105; a new reader must fall back to normal when reopening it. +- A residual `.trim.idx` must not be used after trim metadata is lost, and its space should be reclaimed after the DMFile is naturally rewritten. +- The trim index's pack-mark count does not equal `pack_count`, or bits 3 through 7 are nonzero after load (hard-fail / reject payload). +- Metadata or index-subfile checksum mismatch (hard-fail; not soft-fallback). +- Local disk and disaggregated storage. +- Reopening merged subfiles, clone, restore, GC, and segment replacement. +- Online switching and fallback of the read/write settings. + +### Query-Result Tests + +Run the same dataset with: + +```text +trim reads disabled +trim reads enabled +forced fallback to normal +``` + +Compare complete result sets rather than only row counts. Cover SELECT, aggregation, TopN, LIMIT, concurrent reads, partitioned tables, and multiple DMFiles with mixed versions. + +### Performance Tests + +Construct data with: + +- Pack size 8,192. +- Normal timestamps covering 90 days with temporal locality. +- One out of every 10,000 rows using the 2100 sentinel, uniformly distributed. +- Queries covering the latest consecutive three hours. + +Compare: + +```text +ordinary min-max +ordinary + trim disabled +trim enabled +no-outlier baseline +``` + +Collect at least: + +- Counts of RS none/some/all packs. +- DMFile scanned/skipped rows and bytes. +- Index load bytes/time/cache hits. +- Query p50/p95/p99 latency. +- Merge delta / compact write throughput and CPU. +- DMFile data, index, and metadata sizes. + +Do not use the theoretical 403x pack reduction as a hard acceptance threshold because it depends on temporal locality. The hard requirements are identical query results, no significant regression for workloads without outliers, and scanned rows for the target dataset that are clearly close to the no-outlier baseline. + +## Risks and Mitigations + +### Interpreting an Old Index Using the Current Default Interval + +Risk: The effective interval changes during product evolution, and a new reader produces a false negative from an old trim index. + +Mitigation: Persist the actual bounds for every column in every DMFile. Eligibility uses only the stored range, and the V1 format always uses a half-open interval. Metadata and the index payload containing pack marks are atomically published with the same immutable DMFile generation. Existing checksums and structural validation detect corruption or incorrect combinations. + +### Failure to Adjust `None` or `All` Using Directional Flags + +Risk: After trim ignores outliers, incorrectly retaining `None` can omit matching trimmed rows. Incorrectly retaining `All` can allow non-matching trimmed rows to bypass row-level filtering. + +Mitigation: Persist per-column, per-pack low/high flags. Downgrade `All` to `Some` when a trimmed value that must not match exists, and downgrade `None` to `Some` when a trimmed value that must match exists. Add dedicated result-consistency tests. + +### Incomplete Query-Domain Analysis + +Risk: Incorrect predicate classification can reverse the matching direction of low/high trimmed values or incorrectly apply trim to an OR query. + +Mitigation: In the first version, allowlist equality, IN, top-level AND bounded ranges, and one-sided ranges whose bounds are within `E`. Explicitly test all four comparison directions and packs containing both low and high values. All other expressions fall back. + +### Mixed-Version Read Failure + +Risk: An old reader cannot ignore field 105 or the additional merged subfile and therefore cannot open the DMFile. + +Mitigation: Trim is not added to `indexes = 104`; an old reader only needs to ignore the new optional field. The ordinary `.idx` and raw PackStat remain unchanged. Writes remain disabled until new-write/old-read and mixed-version reopen tests pass. + +### Trim Metadata Lost During an Old-Version Rewrite + +Risk: An old node converts protobuf into an old C++ `ColumnStat` without field 105 and later rewrites the metadata during operations such as local-index building, losing trim metadata and leaving an unreachable `.trim.idx`. + +Mitigation: A new reader must fall back to the ordinary min-max when metadata is absent. Record trim-metadata-lost and orphan metrics. Cover this path with clone, restore, local-index bump, and GC tests. Orphaned space is reclaimed when the DMFile is later rewritten by merge/compact/GC. This condition may reduce performance only and must not affect results. + +### Independent Field Diverges from the Generic Index Registry + +Risk: Because `trim_minmax_index = 105` is not in `indexes = 104`, existing tools, diagnostics, and generic lifecycle APIs that iterate local indexes will not automatically see trim. + +Mitigation: Provide trim-specific ColumnStat accessors, validation, and diagnostic output. Explicitly exclude trim from DDL index IDs, the asynchronous local-index scheduler, and `getLocalIndex(index_id)`. Consider migration to the unified registry only after the minimum supported reader version can ignore unknown kinds. + +### Write CPU or Cache Overhead + +Risk: The additional index may offset the query benefit, especially for workloads without outliers. + +Mitigation: Generate both indexes in one traversal, omit trim when an entire file has no trimmed value, lazily load normal and trim, and provide independent read/write kill switches. + +### Temporal Bound or Time-Zone Inconsistency + +Risk: The writer and reader disagree on the packed representation of bounds or on `TIMESTAMP` literal conversion. + +Mitigation: Store typed packed bounds in metadata. The reader determines the query domain after existing time-zone conversion. Cover FSP, UTC, offset, and DST in tests. + +### Trim Index and MetaV2 Space Growth + +Risk: The additional trim index increases DMFile size, and many temporal columns still add fixed-size column metadata to MetaV2. + +Mitigation: Low/high flags reuse the trim index's existing pack-mark byte, and MetaV2 stores no per-pack data. Do not write trim indexes or metadata for columns without outliers. Add metrics for index/metadata size and parsing time. + +## Alternatives + +### Use NULL to Represent Unsettled Values + +The current ordinary min-max already excludes NULL, so this is the simplest solution when the application can change its data model. However, existing schemas, application compatibility, and other sentinel-value scenarios may prevent a uniform migration, so this cannot replace a general storage-layer optimization. + +### Reduce the Pack Size + +Reducing the number of rows per pack lowers the probability of outlier contamination, but it increases the number of packs, index size, marks, and read/write scheduling overhead. It also does not eliminate contamination by extreme values. + +### Hard-Code the Ordinary Min-Max to Ignore 2100 + +This would produce incorrect results for queries that target or cover 2100 and cannot generalize to other sentinel values. It is rejected. + +### Use Trim Only as an Additional `None` Gate + +This is the easiest approach to make correct: trim only excludes packs, while all other results come from the ordinary min-max. However, it requires loading both ordinary and trim indexes and cannot restore a correct `All`. By persisting low/high flags in the trim index's pack marks, this design allows trim to safely replace the ordinary min-max for eligible queries and therefore chooses the complete RSResult approach. + +### Store Two Independent Bitmaps + +Separate `has_trimmed_low_bitmap` and `has_trimmed_high_bitmap` values require managing two lengths, tail bits, address calculations, and storage locations. The first version reuses the trim min-max's existing `has_null_marks` byte: bit 0 preserves NULL semantics, while bits 1 and 2 represent low and high. This adds no per-pack byte and stores no bitmap in MetaV2. + +### Make `TrimMinMaxIndex` Inherit from `MinMaxIndex` + +The current `MinMaxIndex` state is private, `read()` always constructs the base class, `checkCmp` is a non-virtual template method, and the query path invokes it statically through `MinMaxIndexPtr`. Adding a virtual destructor, virtual interfaces, protected state, and a derived-class factory solely for inheritance would expand changes to the ordinary min-max hot path and cache without changing the disk layout. The first version therefore rejects inheritance and uses composition: a `MinMaxIndex` payload plus a lightweight trim wrapper. + +### Append Trim Directly to `ColumnStat.indexes` + +This would let trim share a unified registry with vector, inverted, and full-text indexes, but it requires a new index kind. Current old readers call `integrityCheckIndexInfoV2` on every element of `indexes = 104`, and an unknown kind causes DMFile restore to fail. The first version cannot use this option unless a reader generation that ignores and preserves unknown kinds is deployed first and rollback to older versions is no longer required. Option B uses a self-contained `TrimMinMaxIndexProps` directly in field 105, bypassing strict traversal by old readers and avoiding dependencies on DDL local-index metadata. + +## Established Design Boundaries + +- The effective interval is the half-open interval `[1900-01-01, 2099-12-01)`. +- Actual bounds are persisted per column and per DMFile. Readers do not use the current default configuration to interpret old indexes. +- The trim index defines the original `has_null_marks` byte array as `pack_marks`: bit 0 is NULL, bit 1 is low, bit 2 is high, and bits 3 through 7 must be zero in V1. +- Trim metadata uses `ColumnStat.trim_minmax_index = 105`, whose type is directly the self-contained `TrimMinMaxIndexProps`. It is not added to `indexes = 104` and does not modify `DMFileIndexInfo`, `IndexFilePropsV2`, or `IndexFileKind`. +- The `MergedSubFileInfo` associated with the deterministic `.trim.idx` file name is the sole source of its location and size; the props do not duplicate the file size. +- Soft fallback covers expected absence and logical meta mismatch. A present but structurally invalid `MergedSubFileInfo`, or a corrupt trim payload/checksum, hard-fails under the existing DMFile corruption policy and must not be converted into a silent ordinary-minmax soft fallback. +- Field 105 stores no per-pack flags. Low/high flags reside in the trim index payload together with min/max. +- The trim min-max uses a separate subfile and does not extend the ordinary `.idx`. +- `TrimMinMaxIndex` uses composition rather than inheritance. Generic `MinMaxIndex` produces the raw result, and the trim wrapper/operator applies directional adjustments. +- The first version writes only DMFile V3 / MetaV2. +- The first version uses trim for allowlisted equality, IN, bounded ranges, and one-sided ranges whose bounds are within `E`. Other predicates fall back. +- Low/high flags determine whether trimmed values must match or must not match and conservatively adjust `None` and `All`. +- Old files are not backfilled; coverage grows through natural rewrites. diff --git a/docs/design/2026-08-31-trim-minmax-for-date-types-columnar.md b/docs/design/2026-08-31-trim-minmax-for-date-types-columnar.md new file mode 100644 index 00000000000..cbd5198078c --- /dev/null +++ b/docs/design/2026-08-31-trim-minmax-for-date-types-columnar.md @@ -0,0 +1,697 @@ +# Trim Min-Max Index for DATE / DATETIME / TIMESTAMP in Next-Gen Columnar (CSE) + +- Status: Accepted +- Date: 2026-08-31 +- Last updated: 2026-09-04 +- Related: [2026-07-14-trim-minmax-for-date-types.md](./2026-07-14-trim-minmax-for-date-types.md) (DeltaMerge / DMFile) +- Primary change surface: `contrib/cloud-storage-engine` (`kvengine` columnar) + +## Summary + +This document ports the DeltaMerge `trim_minmax` optimization to TiFlash **next-gen columnar** mode (`ENABLE_NEXT_GEN_COLUMNAR=ON`), where pack-level min-max and rough-check live in Cloud Storage Engine (CSE) rather than in DMFile. + +The problem is identical: sparse sentinel timestamps such as `2100-01-01 00:00:00` inflate ordinary pack min-max and destroy pruning for narrow recent-time predicates. The correctness foundation (effective interval `E`, trimmed low/high marks, per-predicate eligibility, conservative `None` correction) is intentionally the same as the DeltaMerge design. + +The CSE design adopts these decisions: + +1. Reuse the same default half-open effective range `E = [1900-01-01 00:00:00, 2099-12-01 00:00:00)` and the same packed `u64` temporal encoding already used by CSE columnar values. +2. Keep the ordinary L2 `MinMaxIndex` unchanged. Build an optional **trim** min-max for supported temporal user columns during the same pack traversal. +3. Persist trim as an **optional trailer** appended after the ordinary min-max payload inside the existing `ColumnMeta.compressed_min_max_pack`. Old readers parse only the ordinary prefix and ignore trailing bytes; they do not need a new `ColumnMeta` field layout. +4. Encode per-pack flags in a single unified `pack_marks` byte (same layout in memory and in the TRMM trailer): bit 0 = null, bit 1 = ordinary has_value, bit 2 = trimmed low, bit 3 = trimmed high, bit 4 = trim has_value (in-range); bits 5..7 reserved (`0xE0`). The ordinary **prefix** still expands null / ordinary-has_value into separate legacy byte arrays for MinMax compatibility; the trailer stores only trim values + this unified `pack_marks` array (no separate trailer `has_value_marks`). +5. Perform trim eligibility and rough-check selection entirely inside CSE `FilterOperator` (Rust). TiFlash RN continues to push tipb filters and apply TIMESTAMP literal UTC normalization; it does not load CSE min-max bytes. +6. CSE rough-check results remain `None` / `Some` / `Unknown` and only skip pack I/O. There is **no** `All` that skips row-level filtering on the RN. Therefore CSE only needs `None β†’ Some` correction when a trimmed value must match; the DeltaMerge `All β†’ Some` path is not required for v1. +7. Write trim only on **columnar L2** (same gate as ordinary min-max). L0/L1, non-temporal columns, disabled config, and ineligible predicates always fall back to ordinary min-max (or `Some` when no index exists). +8. Gate trim **reads** via `TableScanCtx::enable_trim_minmax` (plumbed from TiFlash `dt_enable_trim_minmax`). L2 writes always build trim indexes for supported temporal columns when outliers exist. +9. When trim reads are enabled, flatten top-level `And` and merge same-column one-sided temporal compares into a CSE **`DateRange`** leaf (DeltaMerge `normalizeTemporalRangesForTrim` analogue), so bounded ranges use `EqualityOrInOrBounded` correction instead of composing independent one-sided corrections. + +## Background + +### Current CSE Columnar Min-Max + +Relevant code: + +- `contrib/cloud-storage-engine/components/kvengine/src/table/columnar/columnar.rs` β€” `MinMaxIndex`, `ColumnMeta`, `TableMeta` +- `.../columnar/builder.rs` β€” `ColumnarColumnBuilder::finish_pack` / `compute_min_max` / `finish_min_max_pack` +- `.../columnar/temporal_minmax.rs` β€” `TemporalMinMaxIndex`, trailer parse/write, trim rough-check +- `.../columnar/filter.rs` β€” tipb β†’ `FilterOperator` β†’ `rough_check` +- `.../columnar/reader.rs` β€” skip packs with `FilterOpResult::None` + +Write path (L2 only, `level == MAX_COLUMNAR_LEVEL`): + +```text +ColumnarTableBuilder.append + -> ColumnarColumnBuilder::finish_pack(del_marks) + -> has_null = any (null && !deleted) + -> compute_min_max / compute_temporal_min_max + -> MinMaxIndex or TemporalMinMaxIndex push min/max + marks + -> finish_min_max_pack + -> ordinary prefix (+ optional TRMM trailer) -> compress_pack + -> ColumnMeta.compressed_min_max_pack +``` + +On-disk ordinary payload (then LZ4-compressed into `compressed_min_max_pack`): + +```text +ColumnBuffer(length = 2 * num_packs) // min at 2i, max at 2i+1 +has_null_marks[num_packs] // currently 0/1 only +has_value_marks[num_packs] // currently 0/1 only +``` + +`ColumnMeta` layout: + +```text +PackOffsets +col_info_len | tipb::ColumnInfo +min_max_idx_len | compressed_min_max_pack // 0 => no index +column_props_len | column_props // currently always length 0; + // parse hard-skips 4 bytes +``` + +Query path: + +```text +TiFlash StorageDisaggregatedColumnar + -> normalize TIMESTAMP compare literals to UTC + -> serialize tipb filters to CSE +CSE TableScanCtx::to_filter_operator + -> parse tipb And(...) + -> if enable_trim_minmax: normalize_temporal_ranges_for_trim +CSE FilterOperator::rough_check(TableMeta) + -> per pack: ordinary / temporal / DateRange checks -> FilterOpResult::{None,Some,Unknown} +ColumnReader::load_pack + -> None: skip I/O + -> Some/Unknown: load pack +TiFlash RN + -> always applies row-level filters (CSE does not claim All) +``` + +### Problem Scenario + +Same as the DeltaMerge design: with pack size 8,192 and one `2100-01-01` sentinel per 10,000 rows, about 56% of packs have `max = 2100`, so narrow recent-range predicates lose most ordinary min-max pruning power. + +Independently evaluating `t >= L` and `t <= U` as one-sided trim leaves is not enough for BETWEEN-shaped queries: each leaf may apply directional `Noneβ†’Some` correction, and their conjunction keeps packs that a true bounded range (`Q βŠ† E`) can prove empty. DeltaMerge avoids this by normalizing opposite one-sided bounds into `DateRange` with `EqualityOrInOrBounded`; CSE must do the same on the rough-check tree. + +### Why a Separate Columnar Design + +| Dimension | DeltaMerge | CSE columnar | +| --- | --- | --- | +| Owner of index I/O | TiFlash C++ `DMFilePackFilter` | CSE Rust `FilterOperator` / `ColumnReader` | +| Storage | Separate `.idx` / `.trim.idx` + `ColumnStat` protobuf | Embedded `compressed_min_max_pack` in `ColumnMeta` | +| Rough result | `RSResult` with `All` (may skip RN row filter) | `FilterOpResult` without `All`; RN always filters | +| Index levels | Every stable DMFile | **L2 only** | +| Predicate parse | `FilterParser` + `DateQueryDomain` | tipb tree in `filter.rs` (+ `DateRange` normalize) | +| TIMESTAMP TZ | `FilterParser::convertFieldWithTimezone` | RN `normalizeTimestampCompareDateTimeLiteralToUTC`; CSE has `TODO: timezone` | + +Porting the DMFile protobuf / MergedSubFileInfo plan unchanged would not fit CSE's column-meta embedding and rolling-upgrade constraints. + +## Terminology + +| Term | Definition | +| --- | --- | +| ordinary min-max | Existing per-pack CSE `MinMaxIndex` over all non-NULL, non-deleted values | +| effective date range `E` | Persisted half-open interval `[lower, upper)` used when building a trim index | +| trim value | Value in `D ∩ E` that participates in trim min/max | +| trimmed value | Non-NULL, non-deleted value outside `E` | +| `pack_marks` | Unified per-pack `UInt8` (memory + TRMM trailer): null / ordinary_has_value / trimmed_low / trimmed_high / trim_has_value; bits 5..7 reserved | +| trim-eligible | Predicate for which low/high trimmed values have uniform match semantics vs the stored `E` | +| trailer | Bytes appended after the ordinary min-max payload inside one compressed min-max blob | +| `TemporalMinMaxIndex` | In-memory ordinary + optional trim view for a temporal column (parsed from ordinary prefix + TRMM trailer) | +| `DateRange` | Rough-check-only leaf for a temporal interval or one-sided bound (`FilterType::DateRange`); DM alias `DM::DateRange` | +| top-level And flatten | Collapse nested `And` above `Or` / `Not` / compares before merging temporal bounds | + +## Goals + +1. Restore L2 pack pruning for DATE / DATETIME / TIMESTAMP predicates when sparse outliers pollute ordinary min-max. +2. Preserve query correctness: never drop a matching pack via false `None`; never claim stronger certainty than CSE's `FilterOpResult` model allows. +3. Keep "new write / old read" safe without requiring old CSE binaries to understand new `ColumnMeta` fields. +4. Soft-fallback to ordinary min-max for old files, missing trailers, unknown trim versions, and ineligible predicates. +5. Bound write and metadata overhead; omit trim when a column has no trimmed value in the file. +6. Expose enough metrics (CSE + optional RN `ColumnarScanContext`) to validate selection, fallback, and pruning gain. +7. For BETWEEN-shaped tipb (`col >= L AND col <= U` and GT/LT variants), prune packs that only survive via independent one-sided outlier correction. + +## Non-Goals + +- Changing TiDB / TiFlash SQL semantics or temporal type semantics. +- DDL or user-visible index configuration of `E`. +- Writing trim for columnar L0/L1. +- Supporting `TIME` / duration / string / numeric columns in v1. +- Introducing CSE `All` / skipping RN row-level filters in v1. +- Actively rewriting historical columnar files; coverage grows via L2 compaction / major. +- Implementing full timezone conversion inside CSE in v1 (continue relying on RN TIMESTAMP normalization). +- Unifying DeltaMerge and CSE on-disk formats; only share semantics and packed bound constants. +- Rewriting ranges under `Or` / `Not`, or inventing a general predicate simplifier beyond DateRange normalize. + +## Correctness Foundation + +The set algebra is identical to the DeltaMerge design. + +Let `D` be the ordinary min-max value set of a pack and `E` the stored effective range. Trim represents `D_trim = D ∩ E`. For a non-NULL query domain `Q`: + +- If `Q βŠ† E`, then `D ∩ Q = D_trim ∩ Q`, so trim may safely prove emptiness (`None`). +- Trim must not treat "all in-range values match" as pack-level certainty beyond CSE's model. CSE has no `All`, and RN always filters, so the dangerous DeltaMerge `All` path does not exist here. +- For one-sided ranges with the finite bound in `E`, directional trimmed flags are still required: + +| Predicate | Low trimmed | High trimmed | +| --- | --- | --- | +| `col >= T` / `col > T` | Never matches | Always matches | +| `col <= T` / `col < T` | Always matches | Never matches | + +Correction for CSE v1: + +| Raw trim result | Condition | Final | +| --- | --- | --- | +| `None` | `trimmed_match_exists = false` | `None` | +| `None` | `trimmed_match_exists = true` | `Some` | +| `Some` / `Unknown` | any | unchanged | + +Flags never upgrade certainty; they only prevent false negatives on one-sided predicates. + +### AND / OR Composition + +CSE composes leaf results with `&` / `|`. Soundness requires **per-leaf** eligibility against the **stored** `E`, not a column-global choice: + +```text +(t = 2020) OR (t = 2200) +``` + +`t = 2200` is not trim-eligible for `E = [1900, 2099-12)` and must use ordinary min-max. Otherwise an empty `D_trim` could return `None` and drop a matching pack from the `Or`. + +**Bounded ranges must not stay as independent one-sided leaves.** For a pack with in-range history plus a high sentinel and query `L <= t <= U` with `Q βŠ† E`: + +| Leaf | Trim raw | Correction | Result | +| --- | --- | --- | +| `t >= L` | often `None` | `TRIMMED_HIGH` β†’ `Some` | keep | +| `t <= U` | often `Some` | n/a | keep | +| `And` of the two | | | **keep** (too permissive) | + +No row may satisfy both bounds, yet the conjunction retains the pack. Therefore, when trim reads are enabled, top-level `And` of opposite one-sided temporal compares on the same column **must** be normalized into a `DateRange` leaf with `EqualityOrInOrBounded` (outliers never force `Noneβ†’Some`). Not rewriting under `Or` / `Not` remains the conservative rule (same as DeltaMerge). + +## Design + +### Overall Architecture + +```text +Columnar L2 write + -> ordinary MinMaxIndex (unchanged semantics) + -> TemporalMinMaxIndex builder for Date/DateTime/Timestamp(/NewDate) + -> if any trimmed value in column: + append trim trailer to ordinary payload + compress once into compressed_min_max_pack + else: + write ordinary payload only (no trailer) + +TiFlash RN + -> normalize TIMESTAMP literals to UTC (existing) + -> plumb dt_enable_trim_minmax into TableScanCtx + -> push tipb filters to CSE + +CSE FilterOperator + -> parse tipb + -> if enable_trim_minmax: normalize_temporal_ranges_for_trim + (flatten top-level And; merge Ge/Gt/Le/Lt per col into DateRange) + -> per leaf / DateRange, per pack: + if trim-eligible for stored E and trailer present: + raw trim check (+ None->Some correction by predicate class) + else: + ordinary min-max + -> And/Or/Not compose FilterOpResult as today +``` + +### Effective Date Range + +Default (format version 1): + +```text +[1900-01-01 00:00:00, 2099-12-01 00:00:00) +``` + +Bounds are stored as little-endian packed `u64`, matching CSE's columnar temporal encoding (`decode_v2_u64` β†’ 8-byte LE) and DeltaMerge `MyDate` / `MyDateTime::toPackedUInt` for the same calendar values. + +- `DATE` / `NewDate`: packed date. +- `DATETIME` / `TIMESTAMP`: packed datetime (TIMESTAMP values are stored/compared as the UTC packed form already present in columnar data after RN normalization of literals). + +Readers must use **persisted** bounds from the trailer, never the process default, when deciding eligibility. + +### On-Disk Layout: Trim Trailer + +`compressed_min_max_pack` for a temporal column is one LZ4 blob. After decompress, CSE writes / parses: + +```text +[ ordinary MinMax prefix ] // legacy-compatible + ColumnBuffer(length = 2 * num_packs) // ordinary min at 2i, max at 2i+1 + has_null_marks[num_packs] // 0/1 bytes (expanded from pack_marks NULL) + has_value_marks[num_packs] // 0/1 bytes (expanded from ORDINARY_HAS_VALUE) + +[ optional TRMM trailer ] // only if the column has any trimmed outlier + magic u32 LE = 0x4D4D5254 // 'TRMM' + format_version u32 LE = 1 + lower_bound u64 LE // packed; half-open E start + upper_bound u64 LE // packed; half-open E end (exclusive) + pack_count u64 LE // must equal num_packs + trim_payload: + ColumnBuffer(length = 2 * num_packs) // trim min at 2i, max at 2i+1 + pack_marks[pack_count] // unified marks (see Pack Marks); + // bits 5..7 must be 0 +``` + +This matches `TemporalMinMaxIndex::{write_ordinary_prefix_to, write_trailer_to, parse_ordinary_prefix, try_parse_trailer_into}`. + +Constraints: + +1. Ordinary prefix byte layout and semantics are unchanged so old readers keep working. Temporal writers expand unified `pack_marks` into separate `has_null_marks` + `has_value_marks` for that prefix only. +2. `format_version = 1` means half-open `[lower, upper)` and the trim_payload layout above. Unknown versions β†’ soft-ignore trailer (keep ordinary; leave trim disabled). Bumping TRMM version must not change ordinary-prefix layout (`MAINTAINER_GUIDE` Β§17.3). +3. Trailer `pack_count` must equal the column's pack count (and the trim `ColumnBuffer` length / 2). +4. `lower_bound < upper_bound`. +5. Reserved pack-mark bits 5..7 (`RESERVED_MASK = 0xE0`) must be zero in the trailer; non-zero is a hard `InvalidPackMarks` parse error for a claimed v1 trailer. +6. If the file has no trimmed value for the column, omit the trailer entirely (ordinary-only blob). +7. Ordinary and trim are published atomically in one compressed min-max blob with the immutable columnar file. +8. Trailer `pack_marks` are authoritative after a successful parse: they replace any ordinary-only bits reconstructed from the prefix (same unified layout as in memory). +9. Trim serialization is **trim-specific**: do not call ordinary `MinMaxIndex::write_to` for the trailer, because that would emit another legacy `has_value_marks` array the trailer format does not use. + +**Soft fallback** (use ordinary / `Some`, trim disabled): missing trailer, bad/absent magic, unknown `format_version`, empty remaining buffer after ordinary parse. + +**Hard fail** (existing columnar corruption policy): LZ4 / checksum failure on the compressed min-max pack itself; corrupt claimed-v1 trailers (`TooShort`, `InvalidBounds`, `PackCountMismatch`, `InvalidPackMarks`). Do not reinterpret a corrupt compressed blob as "no index." + +#### Rejected metadata alternatives + +| Alternative | Why rejected for v1 | +| --- | --- | +| Put props/index into `column_props` | Current `ColumnMeta::parse` hard-skips only 4 bytes; non-zero props break old readers mid-`TableMeta` | +| New length-prefixed blob after `column_props` | Same sequential-parse breakage for old readers | +| Separate object-store file | Extra DFS object and GC coupling; unnecessary given embedded min-max | +| Replace ordinary with trim-only | Breaks ineligible predicates that need full `D` | + +Fixing `column_props` length parsing is still desirable as cleanup, but must not be the trim compatibility vehicle in v1. + +### Pack Marks + +Unified `pack_marks[i]` (module `pack_mark` in `temporal_minmax.rs`), identical in memory and in the TRMM trailer: + +| Bit | Mask | Name | Meaning | +| --- | --- | --- | --- | +| 0 | `0x01` | `NULL` | pack has at least one non-deleted NULL | +| 1 | `0x02` | `ORDINARY_HAS_VALUE` | pack has at least one non-null value (including outliers) | +| 2 | `0x04` | `TRIMMED_LOW` | pack has at least one non-null value `< lower_bound` | +| 3 | `0x08` | `TRIMMED_HIGH` | pack has at least one non-null value `>= upper_bound` | +| 4 | `0x10` | `TRIM_HAS_VALUE` | pack has at least one non-null value in `[lower_bound, upper_bound)` | +| 5..7 | `0xE0` | reserved | must be 0; non-zero β†’ `InvalidPackMarks` | + +On the ordinary **prefix** only, bits 0 and 1 are expanded into separate legacy arrays (`has_null_marks` / `has_value_marks` as 0/1 bytes) so old MinMax readers stay compatible. The trailer does **not** repeat those arrays; it stores the unified `pack_marks` once next to the trim `ColumnBuffer`. + +Accessors: + +```text +has_null = (mark & 0x01) != 0 // NULL +ordinary_has_value = (mark & 0x02) != 0 // ORDINARY_HAS_VALUE +has_trimmed_low = (mark & 0x04) != 0 // TRIMMED_LOW +has_trimmed_high = (mark & 0x08) != 0 // TRIMMED_HIGH +has_value (trim) = (mark & 0x10) != 0 // TRIM_HAS_VALUE +``` + +Do not infer trim `has_value` from whether the min/max `ColumnBuffer` slots are null: the minmax buffer's `nullable` flag follows the column's nullability, so NotNull columns are not a reliable signal. + +### In-Memory Model + +Prefer a unified temporal index that holds ordinary + optional trim views: + +```rust +struct TemporalMinMaxIndex { + ordinary_values: ColumnBuffer, // 2 slots per pack + trim_values: ColumnBuffer, // 2 slots per pack (meaningful when trailer present) + pack_marks: Vec, // NULL | ORDINARY_HAS_VALUE | LOW | HIGH | TRIM_HAS_VALUE + lower_bound: u64, + upper_bound: u64, + format_version: u32, + // has_trim_trailer / parse state as needed +} + +struct ColumnMeta { + // existing fields... + min_max: Option, // non-temporal (or legacy ordinary-only) + temporal_min_max: Option, // temporal columns; trailer optional +} +``` + +Raw comparison helpers may still share logic with `MinMaxIndex::check_*`, but trim payload parse/write must be dedicated so trim `has_value` is read from / written to bit 4 (`TRIM_HAS_VALUE`) of `pack_marks`. + +`ColumnMeta::parse` flow: + +1. Decompress `compressed_min_max_pack` if `min_max_idx_len > 0`. +2. For temporal columns, parse ordinary prefix into `TemporalMinMaxIndex`; if remaining bytes look like a valid trailer, attach the trim view (replacing `pack_marks` with trailer marks); else leave trailer absent (`has_trim_trailer() == false`). +3. For non-temporal columns, parse ordinary `MinMaxIndex` as today. + +### Write Path + +Extend `ColumnarColumnBuilder` so that when `need_min_max && is_supported_temporal(tp)`: + +```text +for each row in pack: + if deleted: continue + if null: + mark |= NULL // 0x01 + continue + mark |= ORDINARY_HAS_VALUE // 0x02 + ordinary.update_minmax(value) + if lower <= value < upper: + trim.update_minmax(value) + mark |= TRIM_HAS_VALUE // 0x10 + else if value < lower: + mark |= TRIMMED_LOW // 0x04 + else: + mark |= TRIMMED_HIGH // 0x08 + +// after the pack loop: +// append ordinary / trim min-max slots (null placeholders when no value) +// push unified mark into pack_marks +``` + +Single traversal; do not scan the pack twice. + +On `finish_min_max_pack`: + +1. Serialize ordinary payload. +2. If the column-level "any trimmed" flag is set, append trailer with trim payload and bounds. +3. Compress once into `compressed_min_max_pack`. + +No trim for handle / version columns, non-temporal types, L0/L1, or empty tables. Trim generation is not gated by a runtime switch; only the read path is configurable. + +### Query-Domain Analysis (CSE) + +Implement eligibility in `filter.rs` next to `CompareOperator` / `DateRangeCompare`. + +Supported trim-eligible forms (after tipb parse and DateRange normalize): + +```text +col = T +col IN (T1, T2, ...) +col >= L AND col <= U (and GT/LT variants) // as DateRange, EqualityOrInOrBounded +col >= L / col > L // DateRange or one-sided leaf, LowerBounded +col <= U / col < U // DateRange or one-sided leaf, UpperBounded +``` + +Rules: + +- Equality / IN / bounded range: require `Q βŠ† stored E`. +- One-sided: finite bound in `E`; pass predicate class into correction. +- No trim for `NotEqual`, `NotIn`, `IsNull`, `Like`, casts, functions, or branches under `Or`/`Not` that are not independently eligible leaves. +- Per-leaf / DateRange re-check at rough-check time against stored trailer bounds. + +`NotEqual` / `NotIn` continue to use ordinary min-max only. + +DateRange normalize details are Phase D. + +### DateRange Normalize (Read Path) + +Mirror DeltaMerge `normalizeTemporalRangesForTrim`, implemented in CSE only. + +**When:** end of `TableScanCtx::to_filter_operator`, **only if** `enable_trim_minmax` is true. Trim off β†’ no rewrite (ordinary path unchanged). + +**Representation:** + +```text +FilterType::DateRange +DateRangeCompare { + col_id, + lower / upper: Option, + lower_inclusive / upper_inclusive: bool, + predicate_class: TrimPredicateClass, // derived at normalize time +} +``` + +Bound mapping (same as DM): + +| Leaf | BoundSide | +| --- | --- | +| `GreaterEqual` | lower inclusive | +| `Greater` | lower exclusive | +| `LessEqual` | upper inclusive | +| `Less` | upper exclusive | + +Multiple bounds on the same side: keep the **stronger** bound (larger lower / smaller upper; on tie prefer exclusive). + +**Algorithm (`normalize_temporal_ranges_for_trim`):** + +```text +1. Flatten top-level And into leaves (iterative; preserve left-to-right order). +2. For each leaf: + - If Ge/Gt/Le/Lt on a supported temporal column: accumulate BoundAccumulator[col_id]. + - Else: keep as-is (Equal, In, Or, Not, non-temporal, Unsupported, …). +3. For each accumulator: + - On decode failure or empty bounds: append originals unchanged. + - Else emit DateRange with class: + lower && upper -> EqualityOrInOrBounded + lower only -> LowerBounded + upper only -> UpperBounded +4. Rebuild And(kept) (or single leaf). +``` + +Hard rules: + +- Do **not** recurse into `Or` / `Not` when flattening. +- Do **not** merge Equal / In into DateRange. +- Temporal gate uses `is_supported_temporal_type` (Duration stays out, matching trim write support). +- Resolve column type from `ParseCtx.columns` / `to_filter_operator` column list by `col_id`. + +**Rough-check:** evaluate lower and upper jointly against trim (or ordinary) min/max with correct inclusivity; apply `Noneβ†’Some` correction using the DateRange's `predicate_class`. Bounded ranges therefore do not keep packs solely because of high/low outliers. + +**Fallback:** undecodable bounds β†’ keep original leaves; no trailer / ineligible endpoints β†’ ordinary joint range check; nested And under Or β†’ untouched (may forgo benefit; never wrong). + +### Per-File / Per-Column Selection + +```text +if !trim_read_enabled -> ORDINARY +else if no trim trailer -> ORDINARY +else if unsupported format_version -> ORDINARY +else if props inconsistent -> ORDINARY +else if !query_domain.is_trim_eligible -> ORDINARY +else -> TRIM (+ correction) +``` + +L0/L1 columns have `min_max = None` today and already return `Some`; trim does not change that. + +### NULL, Delete, MVCC + +Align with ordinary CSE min-max: + +- Deleted rows excluded from ordinary and trim; they do not set low/high bits. +- Non-deleted NULL sets `NULL` only; NULL is not a trimmed value and does not set `ORDINARY_HAS_VALUE` / `TRIM_HAS_VALUE`. +- `TRIM_HAS_VALUE` (bit 4) is set only when at least one in-range non-deleted value exists; otherwise raw trim checks treat the pack as empty for min/max. +- Pack MVCC stats (`PROP_KEY_PACK_MVCC_STATS`) remain independent of rough-check. + +### Configuration and Switches + +Read-side only. `TableScanCtx` (propagated into `FilterOperator` via `ParseCtx`): + +```text +enable_trim_minmax: bool = false // read path only +``` + +- **Write:** L2 columnar builds always maintain trim index state for supported temporal columns. If a column has any trimmed value in the file, the TRMM trailer is emitted during `finish_min_max_pack`. No build-time kill switch. +- **Read:** when `enable_trim_minmax` is false, skip DateRange normalize, ignore trim trailers, and use ordinary min-max only. When true, normalize top-level And into DateRange where applicable, then apply per-predicate eligibility and trim rough-check correction. +- Default read is disabled; enable gradually on canary nodes via TiFlash `dt_enable_trim_minmax` plumbed into `TableScanCtx::with_enable_trim_minmax`. +- `E` is not runtime-configurable in v1; only persisted bounds matter for eligibility. +- No separate config key for DateRange normalize; it shares the same kill switch. + +### Observability + +Extend CSE `ColumnarRuntimeStats` and, when useful, tipb `ColumnarScanContext` / C++ `ColumnarScanContext`: + +```text +trim_minmax_selected_packs +trim_minmax_none_packs +trim_minmax_none_downgraded_packs +trim_minmax_fallback_count{reason} +trim_minmax_write_bytes +``` + +Fallback reasons at least: `disabled`, `no_trailer`, `unsupported_version`, `predicate_boundary_outside_range`, `unsupported_expression`, `metadata_mismatch`. + +Existing `rough_check_{total,selected,skipped,unknown}_packs` remain the primary pruning counters for DateRange benefit as well. Optional debug logging of DateRange rewrite count is sufficient for v1; no mandatory new metric for normalize itself. + +## Compatibility and Invariants + +### Query-Correctness Invariants + +1. Trim must not make any matching pack disappear (`false None`). +2. Equality / IN / bounded ranges use trim only when `Q βŠ† stored E`. +3. One-sided ranges use trim only when the finite bound is in stored `E` and correction uses low/high marks. +4. Soft-fallback whenever logical selection checks fail. +5. RN row-level filters remain authoritative; CSE does not introduce `All`. +6. Trim mark accessors use only the defined bits: null = bit 0, ordinary_has_value = bit 1, low = bit 2, high = bit 3, trim_has_value = bit 4; bits 5..7 (`0xE0`) must be zero. +7. Eligibility uses stored trailer bounds, not the process default. +8. `Or` / `Not` never gain DateRange rewrite; Equal / In are not merged into DateRange. +9. With trim disabled, the operator tree has no DateRange nodes and rough-check matches ordinary-only behavior for the same tipb. + +### Disk-Format Compatibility + +- Ordinary min-max prefix unchanged β†’ old CSE reads new files. +- Old files without trailer β†’ new CSE uses ordinary only. +- No change to `column_props` length contract in v1. +- No requirement to bump `ColumnarFileFooter.format_version` for trailers (footer version is currently informational). Prefer trailer magic/version for feature detection. +- L2 compaction / major naturally rewrites files; no forced backfill. +- DateRange normalize is read-path only; no disk migration. + +### Rolling Upgrade + +1. Deploy CSE binaries that understand trim trailers (writes begin emitting them immediately on L2 rebuild). +2. Deploy TiFlash / hub that plumbs `dt_enable_trim_minmax` and performs DateRange normalize when enabled. +3. Canary-enable **reads** via `dt_enable_trim_minmax` on a subset of nodes. +4. Verify old binaries still open new L2 files (ordinary prefix only). +5. Expand rollout; keep kill switch for one release cycle. + +## Performance and Resource Overhead + +Per temporal column per pack, the trim trailer stores approximately: + +```text +min + max 16 bytes // MyDateTime / packed u64 +pack_marks 1 byte // NULL | ORDINARY_HAS_VALUE | LOW | HIGH | TRIM_HAS_VALUE +``` + +About **17 bytes/pack/column** uncompressed for the trim trailer payload (ordinary prefix still has its own values + dual mark arrays), then shares one LZ4 frame with ordinary min-max. Versus a DeltaMerge-style trim that keeps a separate trim `has_value_marks` array, CSE folds trim has_value into the same unified byte (and also carries ordinary has_value in that byte so trailer marks can replace the in-memory view). + +- Write: one extra bound compare per non-null non-deleted temporal value in the same traversal. +- Read: when trim-eligible, prefer trim checks; otherwise ordinary. Both indexes are already in memory after decompressing the single blob. Unified `pack_marks` avoids a second mark-array touch on the hot path. +- Normalize: O(n) flatten + per-`col_id` bound merge; negligible vs pack I/O. +- Omit trailer when no trimmed value exists in the column. + +## Phased Implementation + +### Phase A: Format and Parse + +- Define trailer magic / pack-mark helpers in CSE (`NULL|ORDINARY_HAS_VALUE|TRIMMED_LOW|TRIMMED_HIGH|TRIM_HAS_VALUE`, reserved mask `0xE0`). +- Implement trim-specific payload parse/write that stores `ColumnBuffer + unified pack_marks` (no separate trailer `has_value_marks`). +- Teach `ColumnMeta` parse/write to round-trip ordinary+trailer via `TemporalMinMaxIndex`. +- Unit tests: old-prefix-only, trailer present, corrupt v1 trailer hard-fail, unknown version soft-ignore, reserved bits, `TRIM_HAS_VALUE` round-trip. + +### Phase B: Write Path + +- Single-pass ordinary+trim update in `ColumnarColumnBuilder::finish_pack`. +- Gate on L2 + temporal types only (always build when applicable). +- Omit trailer when no trimmed value. +- Compaction/major path inherits via shared builder options. + +### Phase C: Read Path / Eligibility (leaf) + +- Trim eligibility helpers for equality, IN, one-sided compares. +- Per-leaf selection + `Noneβ†’Some` correction in `FilterOperator`. +- Plumb TiFlash `dt_enable_trim_minmax` into `TableScanCtx::with_enable_trim_minmax`. +- Metrics and integration tests with sentinel-contaminated packs for equality / one-sided shapes. + +Phase C alone is **not** sufficient for BETWEEN-shaped tipb; that requires Phase D. + +### Phase D: DateRange AND Normalize + +- Add `FilterType::DateRange` and `DateRangeCompare`. +- Implement `normalize_temporal_ranges_for_trim` at the end of `to_filter_operator` when trim reads are enabled. +- Joint range rough-check + correction by merged `TrimPredicateClass`. +- Unit tests: + - flatten nested top-level `And`; no rewrite under `Or` / `Not` + - stronger-bound merge; inclusive/exclusive endpoints + - bounded range: sentinel-only / inflated packs β†’ `None` with trim ON (not kept via one-sided correction) + - one-sided DateRange / leaf still applies directional correction + - trim OFF: no DateRange nodes +- Update any pre-normalize BETWEEN unit expectations that encoded independent-leaf correction behavior. + +### Phase E: Rollout + +- Canary enable; compare rough-check skip ratios and result correctness vs disabled, including BETWEEN-style workloads. +- Default-enable after soak; retain kill switch (`dt_enable_trim_minmax`). + +## Validation Strategy + +### Unit Tests (CSE) + +Pack shapes: + +```text +all in E +all low / all high / both outliers +normal + 2100 sentinel +NULL only / NULL + normal + outlier +delete mark + normal + outlier +no valid values +``` + +Verify `pack_marks` bit combinations at least: `0x00`, `0x01` (NULL), `0x02` (ORDINARY_HAS_VALUE), `0x04` (TRIMMED_LOW), `0x08` (TRIMMED_HIGH), `0x10` (TRIM_HAS_VALUE), `0x0c` (low+high), `0x13` (null+ordinary+trim has_value), `0x1e` / `0x1f` (ordinary+trim value + outliers Β± null). Reject nonzero bits 5..7 (`0xE0`). + +Rough-check cases (mirror DeltaMerge where applicable): + +```text +pack={2021, 2100}, query=[2020, 2022] -> Some +pack={2100}, query=[2020, 2022] -> None +pack={2100}, query>=2020 -> Some (not None) +pack={1800}, query>=2020 -> None +pack={1800}, query<=2020 -> Some (not None) +pack={2100}, query<=2020 -> None +``` + +DateRange / normalize cases: + +```text +And(Ge, Le) on same temporal col -> one DateRange leaf +And(And(Ge, Le), EqOther) -> flatten; DateRange + Eq +Or(And(Ge, Le), ...) -> inner And not rewritten +pack trim max < L, has TRIMMED_HIGH, + query [L,U] βŠ† E -> None (bounded; not Some via >= correction) +trim OFF -> no DateRange in tree +``` + +### Compatibility Tests + +- New L2 with trailer opened by parser that only reads ordinary prefix. +- Old L2 without trailer opened by new reader. +- Invalid magic / version / bounds / pack_count β†’ ordinary fallback. +- L0/L1 still return `Some` without panic. + +### End-to-End (ENABLE_NEXT_GEN_COLUMNAR) + +- Same SQL result sets with trim disabled / enabled. +- TIMESTAMP with RN timezone normalization + DATETIME/DATE calendar compares. +- Sentinel-contaminated dataset: skipped-pack ratio for narrow recent **ranges** approaches no-outlier baseline (BETWEEN / Ge+Le), not only equality. + +## Risks and Mitigations + +1. **False `None` on one-sided ranges** β€” Persist low/high marks; mandatory `Noneβ†’Some` correction; dedicated tests. +2. **Interpreting old trailers with new default `E`** β€” Eligibility uses persisted bounds only. +3. **OR leaf incorrectly using trim** β€” Per-predicate eligibility at use time; no DateRange rewrite under `Or`. +4. **Trailing-byte parse ambiguity** β€” Strong magic + length/pack_count checks; on failure soft-fallback without touching ordinary result. +5. **Write CPU / size overhead** β€” Single pass; omit trailer when unused; default off. +6. **TIMESTAMP timezone drift** β€” Keep RN UTC normalization; document CSE `ParseCtx` timezone TODO as non-goal for v1; add TIMESTAMP DST tests through RN+CSE. +7. **Divergent DM vs CSE behavior** β€” Share `E`, packed constants, eligibility rules, DateRange normalize rules, and correction tables in design; accept different on-disk containers. +8. **Incorrect exclusive-bound handling in DateRange** β€” Dedicated GT/LT unit cases; packed-u64 endpoint compares shared with existing helpers. +9. **Missing flatten of nested tipb `LogicalAnd`** β€” Tests that wrap `And(Ge, Le)` as a single filter child as well as flat sibling filters. + +## Alternatives + +1. **Only use trim as an extra `None` gate while always loading ordinary** β€” Safer but doubles logic and cannot help when ordinary max is already polluted for complementary checks; rejected in favor of replace-when-eligible (same as DeltaMerge). +2. **Introduce CSE `All` to skip RN filters** β€” Large semantic change across RN/CSE; out of scope. +3. **Build trim on L0/L1** β€” Ordinary min-max is L2-only today; expanding levels is a separate project. +4. **Implement eligibility only in TiFlash C++** β€” CSE owns rough-check; pushing selection to RN would require new tipb annotations and still need CSE correction. Rejected for v1. +5. **Keep a separate `has_value_marks` array in the trim trailer (DeltaMerge-style)** β€” Works, but wastes one byte per pack and an extra array touch. CSE trim already has a dedicated trailer format, so folding trim has_value into `pack_marks` bit 4 (`TRIM_HAS_VALUE`) β€” and carrying ordinary has_value as bit 1 in the same unified byte β€” is preferred. Ordinary **prefix** still expands dual mark arrays for compatibility. +6. **Infer `has_value` from null min/max slots** β€” Unsafe for NotNull columns whose minmax `ColumnBuffer` may not treat empty packs as null the same way; rejected in favor of an explicit bit. +7. **Skip DateRange normalize; rely on leaf And of one-sided trim** β€” Incorrectly retains sentinel-inflated packs for BETWEEN; rejected (Phase D is required for range workloads). +8. **Only merge when both lower and upper exist** β€” Fixes BETWEEN with less code, but diverges from DM and leaves two shapes for one-sided vs two-sided; rejected in favor of full DM parity. +9. **Normalize only inside `prepare_rough_check` without `FilterType::DateRange`** β€” Smaller enum surface, but tree shape is hard to unit-test and drifts from DM's explicit `DateRange`; rejected. +10. **Always normalize even when trim read is off** β€” Extra rewrite cost for no benefit; DM gates on the same switch; rejected. + +## Established Design Boundaries + +- `E = [1900-01-01, 2099-12-01)` half-open; persisted per column trailer. +- Trim trailer appended after ordinary min-max payload inside `compressed_min_max_pack`. +- Ordinary prefix byte-compatible with existing CSE readers; ordinary still uses separate `has_null_marks` + `has_value_marks`. +- Unified `pack_marks` (memory + TRMM): NULL / ORDINARY_HAS_VALUE / TRIMMED_LOW / TRIMMED_HIGH / TRIM_HAS_VALUE; bits 5..7 (`0xE0`) zero; no separate trim `has_value_marks`. +- In-memory temporal columns use `TemporalMinMaxIndex` (ordinary + optional trim view). +- L2-only; temporal types only; read-side `enable_trim_minmax` switch (default off), plumbed from TiFlash `dt_enable_trim_minmax`. +- CSE `FilterOpResult` model unchanged: no `All`; only `Noneβ†’Some` trim correction. +- Per-leaf eligibility; soft-fallback on logical meta problems. +- Top-level And DateRange normalize when trim reads are enabled; no rewrite under `Or` / `Not`. +- No historical backfill; natural L2 rewrite increases coverage. + +## Open Questions + +None that block the design shape above. Follow-ups that may be tracked outside this doc: + +1. ~~Whether to plumb TiFlash `dt_enable_trim_minmax` into `TableScanCtx::with_enable_trim_minmax` automatically.~~ **Decided:** yes; same setting gates leaf trim reads and DateRange normalize. +2. Whether to later fix `column_props` length parsing as independent cleanup and migrate trailers into first-class column props in a future format version. +3. Whether RN should surface trim-specific counters in `EXPLAIN ANALYZE` beyond existing rough-check pack stats.