Skip to content

Commit dfb3e92

Browse files
authored
Statistics for nested columns (#9775)
## Summary - Tracking Issue: #6389 Being able to perform file level pruning on nested columns speeds up queries. ## Changes Adds post order stats to the file stats footer for nested columns. It adds a new `is_nested: bool = false` to the stats set flatbuffer for backward compatibility. Additionally it modifies the way the stats footer is accumulated to use the same AggRef functions that the zone maps use as well. Only goofy thing it it also has to track truncatable stats separately because the aggregate partials don't have a way to express exactness. <!-- ## API Changes Uncomment this section if there are user-facing changes, such as a broken public API or a change in the behavior of an engine integration. A breaking change also needs the `changelog/break` label. --> --------- Signed-off-by: "Thor" <thor.hansen@dash0.com> Signed-off-by: Thor <thor.hansen@dash0.com>
1 parent 91b38f4 commit dfb3e92

27 files changed

Lines changed: 3003 additions & 805 deletions

File tree

‎Cargo.lock‎

Lines changed: 1 addition & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎vortex-array/src/stats/mod.rs‎

Lines changed: 0 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -18,16 +18,6 @@ use vortex_error::VortexExpect;
1818

1919
use crate::expr::stats::Stat;
2020

21-
/// Statistics that are used for pruning files (i.e., we want to ensure they are computed when compressing/writing).
22-
/// Sum is included for boolean arrays.
23-
pub const PRUNING_STATS: &[Stat] = &[
24-
Stat::Min,
25-
Stat::Max,
26-
Stat::Sum,
27-
Stat::NullCount,
28-
Stat::NaNCount,
29-
];
30-
3121
pub fn as_stat_bitset_bytes(stats: &[Stat]) -> Vec<u8> {
3222
let max_stat = u8::from(last::<Stat>().vortex_expect("last stat")) as usize + 1;
3323
// TODO(ngates): use vortex-buffer::BitBuffer

‎vortex-datafusion/src/persistent/format.rs‎

Lines changed: 18 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -45,8 +45,12 @@ use futures::stream;
4545
use object_store::ObjectMeta;
4646
use object_store::ObjectStore;
4747
use vortex::VortexSessionDefault;
48+
use vortex::aggregate_fn::AggregateFnVTableExt;
49+
use vortex::aggregate_fn::EmptyOptions;
50+
use vortex::aggregate_fn::fns::is_constant::IsConstant;
4851
use vortex::array::memory::MemorySessionExt;
4952
use vortex::dtype::DType;
53+
use vortex::dtype::FieldPath;
5054
use vortex::dtype::PType;
5155
use vortex::error::VortexExpect;
5256
use vortex::error::VortexResult;
@@ -605,7 +609,7 @@ impl FileFormat for VortexFormat {
605609
})
606610
});
607611

608-
let (dtype, file_stats, row_count) = match cached_metadata {
612+
let (_dtype, file_stats, row_count) = match cached_metadata {
609613
Some(metadata) => metadata,
610614
None => {
611615
// Not entry - open the file
@@ -642,10 +646,6 @@ impl FileFormat for VortexFormat {
642646
}
643647
};
644648

645-
let struct_dtype = dtype
646-
.as_struct_fields_opt()
647-
.vortex_expect("dtype is not a struct");
648-
649649
// Evaluate the statistics for each column that we are able to return to DataFusion.
650650
let Some(file_stats) = file_stats else {
651651
// If the file has no column stats, the best we can do is return a row count.
@@ -668,12 +668,23 @@ impl FileFormat for VortexFormat {
668668
for field in table_schema.fields().iter() {
669669
// If the column does not exist, continue. This can happen if the schema has evolved
670670
// but we have not yet updated the Vortex file.
671-
let Some(col_idx) = struct_dtype.find(field.name()) else {
671+
let Some((aggregates, stats_dtype)) =
672+
file_stats.get_by_path(&FieldPath::from_name(field.name().as_str()))
673+
else {
672674
// The default sets all statistics to `Precision<Absent>`.
673675
column_statistics.push(ColumnStatistics::default());
674676
continue;
675677
};
676-
let (stats_set, stats_dtype) = file_stats.get(col_idx);
678+
let mut stats_set = aggregates.to_stats_set();
679+
// `IsConstant` has no aggregate behind its stat slot, so `to_stats_set` omits it.
680+
if let Some(is_constant) = aggregates
681+
.get(&IsConstant.bind(EmptyOptions))
682+
.as_exact()
683+
.and_then(|is_constant| bool::try_from(&is_constant).ok())
684+
{
685+
stats_set.set(Stat::IsConstant, Precision::exact(is_constant));
686+
}
687+
let stats_set = &stats_set;
677688

678689
// Update the total size in bytes.
679690
let column_size =

‎vortex-duckdb/cpp/copy_function.cpp‎

Lines changed: 5 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,6 @@
1313
#include "duckdb/main/capi/capi_internal.hpp"
1414
#include "duckdb/main/client_context.hpp"
1515
#include "duckdb/main/connection.hpp"
16-
#include "duckdb/parser/keyword_helper.hpp"
1716
#include "duckdb/parser/parsed_data/create_copy_function_info.hpp"
1817

1918
unique_ptr<FunctionData> copy_to_bind(ClientContext &,
@@ -42,7 +41,7 @@ unique_ptr<FunctionData> copy_to_bind(ClientContext &,
4241
throw BinderException(IntoErrString(error_out));
4342
}
4443
auto cdata = unique_ptr<CData>(reinterpret_cast<CData *>(ffi_bind_data));
45-
return make_uniq<VortexCopyBindData>(std::move(cdata), names);
44+
return make_uniq<VortexCopyBindData>(std::move(cdata));
4645
}
4746

4847
unique_ptr<GlobalFunctionData>
@@ -88,7 +87,7 @@ void copy_to_get_written_statistics(ClientContext &,
8887
gstate.Cast<VortexCopyGlobalState>().written_stats = &statistics;
8988
}
9089

91-
void copy_to_finalize(ClientContext &, FunctionData &bind_data, GlobalFunctionData &gstate) {
90+
void copy_to_finalize(ClientContext &, FunctionData &, GlobalFunctionData &gstate) {
9291
auto &global = gstate.Cast<VortexCopyGlobalState>();
9392
void *const ffi_global = global.ffi_global->DataPtr();
9493

@@ -101,23 +100,14 @@ void copy_to_finalize(ClientContext &, FunctionData &bind_data, GlobalFunctionDa
101100
if (!global.written_stats) {
102101
return;
103102
}
104-
const auto &names = bind_data.Cast<VortexCopyBindData>().column_names;
105103
duckdb_vx_written_file_statistics file_stats;
106104
if (!duckdb_copy_function_get_written_file_statistics(ffi_global, &file_stats)) {
107105
throw InternalException("vortex COPY: written statistics were requested but not produced");
108106
}
109-
if (file_stats.num_columns != names.size()) {
110-
throw InternalException("vortex COPY: %llu statistics columns for %llu written columns",
111-
file_stats.num_columns,
112-
names.size());
113-
}
114107
D_ASSERT(global.written_stats != nullptr);
115108
global.written_stats->row_count = file_stats.row_count;
116109
global.written_stats->file_size_bytes = file_stats.file_size_bytes;
117110
global.written_stats->footer_size_bytes = Value::UBIGINT(file_stats.footer_size_bytes);
118-
// Keyed by top-level column name only. The vortex footer reports one statistics set per
119-
// top-level field, so nested struct/list leaf columns get no statistics here (unlike parquet,
120-
// which recurses to leaf paths). Flat tables are fully covered.
121111
for (idx_t i = 0; i < file_stats.num_columns; i++) {
122112
duckdb_vx_written_column_statistics col_stats {};
123113
duckdb_vx_error col_error = nullptr;
@@ -146,9 +136,9 @@ void copy_to_finalize(ClientContext &, FunctionData &bind_data, GlobalFunctionDa
146136
if (col_stats.has_nan_stat) {
147137
column["has_nan"] = Value::BOOLEAN(col_stats.contains_nan);
148138
}
149-
150-
string quoted = SQLQuotedIdentifier::ToString(names[i]);
151-
global.written_stats->column_statistics.emplace(std::move(quoted), std::move(column));
139+
auto column_key = reinterpret_cast<Value *>(col_stats.column_key)->ToString();
140+
duckdb_destroy_value(&col_stats.column_key);
141+
global.written_stats->column_statistics.emplace(std::move(column_key), std::move(column));
152142
}
153143
}
154144

‎vortex-duckdb/cpp/include/copy_function.hpp‎

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -8,11 +8,9 @@
88
using namespace duckdb;
99

1010
struct VortexCopyBindData final : TableFunctionData {
11-
VortexCopyBindData(unique_ptr<CData> ffi_bind, vector<Identifier> column_names)
12-
: ffi_bind(std::move(ffi_bind)), column_names(std::move(column_names)) {
11+
VortexCopyBindData(unique_ptr<CData> ffi_bind) : ffi_bind(std::move(ffi_bind)) {
1312
}
1413
unique_ptr<CData> ffi_bind;
15-
vector<Identifier> column_names;
1614
};
1715

1816
struct VortexCopyGlobalState final : GlobalFunctionData {

‎vortex-duckdb/cpp/include/table_function.h‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -83,6 +83,8 @@ typedef struct {
8383

8484
// Per-column statistics of a written Vortex file.
8585
typedef struct {
86+
// Owned VARCHAR holding column's dot-separated path.
87+
duckdb_value column_key;
8688
// Owned values, null if absent; the caller must destroy them.
8789
duckdb_value min;
8890
duckdb_value max;

‎vortex-duckdb/src/column_statistics.rs‎

Lines changed: 25 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -1,14 +1,22 @@
11
// SPDX-License-Identifier: Apache-2.0
22
// SPDX-FileCopyrightText: Copyright the Vortex contributors
33

4+
use vortex::aggregate_fn::AggregateFnRef;
5+
use vortex::aggregate_fn::AggregateFnVTableExt;
6+
use vortex::aggregate_fn::EmptyOptions;
7+
use vortex::aggregate_fn::NumericalAggregateOpts;
8+
use vortex::aggregate_fn::fns::max::Max;
9+
use vortex::aggregate_fn::fns::min::Min;
10+
use vortex::aggregate_fn::fns::nan_count::NanCount;
11+
use vortex::aggregate_fn::fns::null_count::NullCount;
12+
use vortex::aggregate_fn::fns::uncompressed_size_in_bytes::UncompressedSizeInBytes;
13+
use vortex::aggregate_fn::fns::uncompressed_size_in_bytes::UncompressedSizeOpts;
414
use vortex::array::match_each_float_ptype;
5-
use vortex::array::stats::StatsSet;
615
use vortex::dtype::DType;
7-
use vortex::dtype::PType;
816
use vortex::error::VortexExpect as _;
917
use vortex::error::VortexResult;
1018
use vortex::expr::stats::Precision;
11-
use vortex::expr::stats::Stat;
19+
use vortex::layout::layouts::file_stats::AggregateStats;
1220
use vortex::scalar::Scalar;
1321
use vortex::scalar::ScalarValue;
1422

@@ -74,33 +82,25 @@ pub struct ColumnStatisticsAggregate {
7482
}
7583

7684
impl ColumnStatisticsAggregate {
77-
pub fn new(stats: &StatsSet) -> Self {
78-
let min = match stats.get(Stat::Min) {
79-
Precision::Exact(min) => Some(min),
80-
_ => None,
81-
};
82-
let max = match stats.get(Stat::Max) {
83-
Precision::Exact(max) => Some(max),
85+
pub fn new(aggregates: &AggregateStats) -> Self {
86+
let exact = |aggregate_fn: AggregateFnRef| match aggregates.get(&aggregate_fn) {
87+
Precision::Exact(value) => value.into_value(),
8488
_ => None,
8589
};
8690

87-
let max_string_length =
88-
if let Precision::Exact(value) = stats.get(Stat::UncompressedSizeInBytes) {
89-
// DuckDB's string length is u32
90-
#[allow(clippy::cast_possible_truncation)]
91-
Some(value.as_primitive().as_u64().vortex_expect("not a u64") as u32)
92-
} else {
93-
None
94-
};
91+
let min = exact(Min.bind(NumericalAggregateOpts::skip_nans()));
92+
let max = exact(Max.bind(NumericalAggregateOpts::skip_nans()));
9593

96-
let has_null = match stats.get(Stat::NullCount) {
97-
Precision::Exact(cnt) => cnt.as_primitive().as_u64().vortex_expect("not a u64") > 0,
98-
_ => true,
99-
};
94+
// DuckDB's string length is u32
95+
#[allow(clippy::cast_possible_truncation)]
96+
let max_string_length = exact(UncompressedSizeInBytes.bind(UncompressedSizeOpts::exact()))
97+
.map(|value| value.as_primitive().as_u64().vortex_expect("not a u64") as u32);
98+
99+
let has_null = exact(NullCount.bind(EmptyOptions))
100+
.is_none_or(|count| count.as_primitive().as_u64().vortex_expect("not a u64") > 0);
100101

101-
let nan_count = stats
102-
.get_as::<u64>(Stat::NaNCount, &PType::U64.into())
103-
.as_exact();
102+
let nan_count = exact(NanCount.bind(EmptyOptions))
103+
.map(|count| count.as_primitive().as_u64().vortex_expect("not a u64"));
104104

105105
Self {
106106
min,

0 commit comments

Comments
 (0)