Skip to content

Commit 0399008

Browse files
committed
feat: add deletion vector compaction planner
Add a snapshot-bound metadata-only planner that selects rewrite candidates by file size and deduplicated file-scoped position-delete pressure. Validate planner inputs and group candidates deterministically within partitions using target-sized groups.
1 parent dc84d6e commit 0399008

7 files changed

Lines changed: 754 additions & 0 deletions

File tree

‎src/iceberg/CMakeLists.txt‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@ set(ICEBERG_SOURCES
2626
catalog/memory/in_memory_catalog.cc
2727
catalog/session_catalog.cc
2828
catalog/session_context.cc
29+
compaction_planner.cc
2930
delete_file_index.cc
3031
deletes/dv_util.cc
3132
deletes/dv_writer.cc

‎src/iceberg/compaction_planner.cc‎

Lines changed: 279 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,279 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing,
13+
* software distributed under the License is distributed on an
14+
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
* KIND, either express or implied. See the License for the
16+
* specific language governing permissions and limitations
17+
* under the License.
18+
*/
19+
20+
#include "iceberg/compaction_planner.h"
21+
22+
#include <algorithm>
23+
#include <cmath>
24+
#include <compare>
25+
#include <limits>
26+
#include <map>
27+
#include <string>
28+
#include <string_view>
29+
#include <tuple>
30+
#include <utility>
31+
32+
#include "iceberg/expression/literal.h"
33+
#include "iceberg/manifest/manifest_entry.h"
34+
#include "iceberg/table_scan.h"
35+
#include "iceberg/util/content_file_util.h"
36+
#include "iceberg/util/data_file_set.h"
37+
#include "iceberg/util/macros.h"
38+
39+
namespace iceberg {
40+
namespace {
41+
42+
struct CanonicalPartitionKey {
43+
int32_t spec_id;
44+
std::string values;
45+
46+
auto operator<=>(const CanonicalPartitionKey&) const = default;
47+
};
48+
49+
struct Candidate {
50+
CompactionFile file;
51+
bool has_delete_pressure;
52+
};
53+
54+
struct PartitionCandidates {
55+
int32_t spec_id;
56+
PartitionValues partition;
57+
std::vector<Candidate> files;
58+
};
59+
60+
Status ValidateConfig(const CompactionPlannerConfig& config) {
61+
ICEBERG_PRECHECK(config.target_file_size_bytes > 0,
62+
"target_file_size_bytes must be greater than zero");
63+
ICEBERG_PRECHECK(std::isfinite(config.min_file_size_ratio) &&
64+
config.min_file_size_ratio >= 0 && config.min_file_size_ratio <= 1,
65+
"min_file_size_ratio must be finite and in [0, 1]");
66+
ICEBERG_PRECHECK(
67+
std::isfinite(config.max_file_size_ratio) && config.max_file_size_ratio >= 1,
68+
"max_file_size_ratio must be finite and at least 1");
69+
ICEBERG_PRECHECK(config.min_input_files > 0,
70+
"min_input_files must be greater than zero");
71+
ICEBERG_PRECHECK(config.delete_file_threshold >= 0,
72+
"delete_file_threshold must not be negative");
73+
ICEBERG_PRECHECK(std::isfinite(config.delete_ratio_threshold) &&
74+
config.delete_ratio_threshold >= 0 &&
75+
config.delete_ratio_threshold <= 1,
76+
"delete_ratio_threshold must be finite and in [0, 1]");
77+
return {};
78+
}
79+
80+
Result<int64_t> CheckedAddNonNegative(int64_t lhs, int64_t rhs,
81+
std::string_view description) {
82+
ICEBERG_PRECHECK(lhs >= 0 && rhs >= 0, "{} must not be negative", description);
83+
ICEBERG_PRECHECK(lhs <= std::numeric_limits<int64_t>::max() - rhs, "{} overflow",
84+
description);
85+
return lhs + rhs;
86+
}
87+
88+
void AppendFramed(std::string& output, std::string_view value) {
89+
output.append(std::to_string(value.size()));
90+
output.push_back(':');
91+
output.append(value);
92+
}
93+
94+
Result<CanonicalPartitionKey> MakePartitionKey(const DataFile& data_file) {
95+
ICEBERG_PRECHECK(data_file.partition_spec_id.has_value(),
96+
"Data file '{}' is missing partition_spec_id", data_file.file_path);
97+
98+
std::string encoded;
99+
encoded.append(std::to_string(data_file.partition.num_fields()));
100+
encoded.push_back(':');
101+
for (const auto& literal : data_file.partition.values()) {
102+
AppendFramed(encoded, literal.type()->ToString());
103+
if (literal.IsNull()) {
104+
encoded.push_back('N');
105+
continue;
106+
}
107+
108+
encoded.push_back('V');
109+
ICEBERG_ASSIGN_OR_RAISE(auto bytes, literal.Serialize());
110+
std::string_view serialized;
111+
if (!bytes.empty()) {
112+
serialized =
113+
std::string_view(reinterpret_cast<const char*>(bytes.data()), bytes.size());
114+
}
115+
AppendFramed(encoded, serialized);
116+
}
117+
return CanonicalPartitionKey{.spec_id = *data_file.partition_spec_id,
118+
.values = std::move(encoded)};
119+
}
120+
121+
Result<CompactionFile> BuildCompactionFile(
122+
const std::shared_ptr<FileScanTask>& scan_task) {
123+
CompactionFile result{.scan_task = scan_task};
124+
const auto& data_file = scan_task->data_file();
125+
ICEBERG_PRECHECK(data_file->record_count >= 0,
126+
"Data file '{}' has negative record count", data_file->file_path);
127+
ICEBERG_PRECHECK(data_file->file_size_in_bytes >= 0,
128+
"Data file '{}' has negative file size", data_file->file_path);
129+
130+
DeleteFileSet applicable_deletes;
131+
for (const auto& delete_file : scan_task->delete_files()) {
132+
ICEBERG_PRECHECK(delete_file != nullptr, "Data file '{}' has a null delete file",
133+
data_file->file_path);
134+
if (delete_file->content != DataFile::Content::kPositionDeletes) {
135+
continue;
136+
}
137+
138+
ICEBERG_ASSIGN_OR_RAISE(auto referenced_file,
139+
ContentFileUtil::ReferencedDataFile(*delete_file));
140+
if (!referenced_file.has_value() || *referenced_file != data_file->file_path) {
141+
continue;
142+
}
143+
144+
if (!applicable_deletes.insert(delete_file).second) {
145+
continue;
146+
}
147+
148+
ICEBERG_PRECHECK(delete_file->record_count >= 0,
149+
"Delete file '{}' has negative record count",
150+
delete_file->file_path);
151+
if (delete_file->IsDeletionVector()) {
152+
ICEBERG_PRECHECK(
153+
delete_file->record_count <= data_file->record_count,
154+
"Deletion vector '{}' cardinality {} exceeds data file '{}' record count {}",
155+
delete_file->file_path, delete_file->record_count, data_file->file_path,
156+
data_file->record_count);
157+
}
158+
159+
ICEBERG_ASSIGN_OR_RAISE(result.file_scoped_delete_count,
160+
CheckedAddNonNegative(result.file_scoped_delete_count, 1,
161+
"File-scoped delete-file count"));
162+
ICEBERG_ASSIGN_OR_RAISE(result.file_scoped_delete_record_count,
163+
CheckedAddNonNegative(result.file_scoped_delete_record_count,
164+
delete_file->record_count,
165+
"File-scoped delete-record count"));
166+
}
167+
168+
result.file_scoped_delete_record_count =
169+
std::min(result.file_scoped_delete_record_count, data_file->record_count);
170+
return result;
171+
}
172+
173+
bool CandidateLess(const Candidate& lhs, const Candidate& rhs) {
174+
const auto& lhs_data = *lhs.file.scan_task->data_file();
175+
const auto& rhs_data = *rhs.file.scan_task->data_file();
176+
return std::tie(lhs_data.file_path, lhs_data.file_size_in_bytes, lhs_data.record_count,
177+
lhs.file.file_scoped_delete_count,
178+
lhs.file.file_scoped_delete_record_count) <
179+
std::tie(rhs_data.file_path, rhs_data.file_size_in_bytes, rhs_data.record_count,
180+
rhs.file.file_scoped_delete_count,
181+
rhs.file.file_scoped_delete_record_count);
182+
}
183+
184+
Result<void> AddToGroup(CompactionGroup& group, Candidate candidate,
185+
bool& has_delete_pressure) {
186+
const auto& data_file = *candidate.file.scan_task->data_file();
187+
ICEBERG_ASSIGN_OR_RAISE(
188+
group.data_file_size_bytes,
189+
CheckedAddNonNegative(group.data_file_size_bytes, data_file.file_size_in_bytes,
190+
"Compaction group data-file size"));
191+
ICEBERG_ASSIGN_OR_RAISE(
192+
group.file_scoped_delete_record_count,
193+
CheckedAddNonNegative(group.file_scoped_delete_record_count,
194+
candidate.file.file_scoped_delete_record_count,
195+
"Compaction group delete-record count"));
196+
has_delete_pressure |= candidate.has_delete_pressure;
197+
group.files.push_back(std::move(candidate.file));
198+
return {};
199+
}
200+
201+
void FinishGroup(CompactionGroup& group, bool has_delete_pressure, size_t min_input_files,
202+
CompactionPlan& plan) {
203+
if (!group.files.empty() &&
204+
(has_delete_pressure || group.files.size() >= min_input_files)) {
205+
plan.groups.push_back(std::move(group));
206+
}
207+
}
208+
209+
} // namespace
210+
211+
Result<CompactionPlan> CompactionPlanner::Plan(
212+
int64_t source_snapshot_id, std::span<const std::shared_ptr<FileScanTask>> scan_tasks,
213+
const CompactionPlannerConfig& config) {
214+
ICEBERG_RETURN_UNEXPECTED(ValidateConfig(config));
215+
const long double min_file_size =
216+
config.target_file_size_bytes * config.min_file_size_ratio;
217+
const long double max_file_size =
218+
config.target_file_size_bytes * config.max_file_size_ratio;
219+
std::map<CanonicalPartitionKey, PartitionCandidates> partitions;
220+
221+
for (const auto& scan_task : scan_tasks) {
222+
ICEBERG_PRECHECK(scan_task != nullptr, "File scan task must not be null");
223+
const auto& data_file = *scan_task->data_file();
224+
ICEBERG_ASSIGN_OR_RAISE(auto partition_key, MakePartitionKey(data_file));
225+
ICEBERG_ASSIGN_OR_RAISE(auto file, BuildCompactionFile(scan_task));
226+
227+
const bool is_small = data_file.file_size_in_bytes < min_file_size;
228+
const bool is_oversized = data_file.file_size_in_bytes > max_file_size;
229+
const bool has_many_delete_files =
230+
config.delete_file_threshold > 0 &&
231+
file.file_scoped_delete_count >= config.delete_file_threshold;
232+
const bool has_high_delete_ratio =
233+
config.delete_ratio_threshold > 0 && data_file.record_count > 0 &&
234+
static_cast<double>(file.file_scoped_delete_record_count) /
235+
data_file.record_count >=
236+
config.delete_ratio_threshold;
237+
const bool has_delete_pressure = has_many_delete_files || has_high_delete_ratio;
238+
const bool should_compact =
239+
is_oversized ? has_delete_pressure : is_small || has_delete_pressure;
240+
if (!should_compact) {
241+
continue;
242+
}
243+
244+
auto partition =
245+
partitions
246+
.try_emplace(std::move(partition_key),
247+
PartitionCandidates{.spec_id = *data_file.partition_spec_id,
248+
.partition = data_file.partition})
249+
.first;
250+
partition->second.files.push_back(
251+
Candidate{.file = std::move(file), .has_delete_pressure = has_delete_pressure});
252+
}
253+
254+
CompactionPlan plan{.source_snapshot_id = source_snapshot_id};
255+
for (auto& [_, partition] : partitions) {
256+
std::ranges::sort(partition.files, CandidateLess);
257+
258+
CompactionGroup group{.partition_spec_id = partition.spec_id,
259+
.partition = partition.partition};
260+
bool has_delete_pressure = false;
261+
for (auto& candidate : partition.files) {
262+
const int64_t file_size = candidate.file.scan_task->data_file()->file_size_in_bytes;
263+
if (!group.files.empty() &&
264+
(group.data_file_size_bytes > config.target_file_size_bytes ||
265+
file_size > config.target_file_size_bytes - group.data_file_size_bytes)) {
266+
FinishGroup(group, has_delete_pressure, config.min_input_files, plan);
267+
group = CompactionGroup{.partition_spec_id = partition.spec_id,
268+
.partition = partition.partition};
269+
has_delete_pressure = false;
270+
}
271+
ICEBERG_RETURN_UNEXPECTED(
272+
AddToGroup(group, std::move(candidate), has_delete_pressure));
273+
}
274+
FinishGroup(group, has_delete_pressure, config.min_input_files, plan);
275+
}
276+
return plan;
277+
}
278+
279+
} // namespace iceberg

0 commit comments

Comments
 (0)