|
| 1 | +/* |
| 2 | + * Licensed to the Apache Software Foundation (ASF) under one |
| 3 | + * or more contributor license agreements. See the NOTICE file |
| 4 | + * distributed with this work for additional information |
| 5 | + * regarding copyright ownership. The ASF licenses this file |
| 6 | + * to you under the Apache License, Version 2.0 (the |
| 7 | + * "License"); you may not use this file except in compliance |
| 8 | + * with the License. You may obtain a copy of the License at |
| 9 | + * |
| 10 | + * http://www.apache.org/licenses/LICENSE-2.0 |
| 11 | + * |
| 12 | + * Unless required by applicable law or agreed to in writing, |
| 13 | + * software distributed under the License is distributed on an |
| 14 | + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
| 15 | + * KIND, either express or implied. See the License for the |
| 16 | + * specific language governing permissions and limitations |
| 17 | + * under the License. |
| 18 | + */ |
| 19 | + |
| 20 | +#include "iceberg/compaction_planner.h" |
| 21 | + |
| 22 | +#include <algorithm> |
| 23 | +#include <cmath> |
| 24 | +#include <compare> |
| 25 | +#include <limits> |
| 26 | +#include <map> |
| 27 | +#include <string> |
| 28 | +#include <string_view> |
| 29 | +#include <tuple> |
| 30 | +#include <utility> |
| 31 | + |
| 32 | +#include "iceberg/expression/literal.h" |
| 33 | +#include "iceberg/manifest/manifest_entry.h" |
| 34 | +#include "iceberg/table_scan.h" |
| 35 | +#include "iceberg/util/content_file_util.h" |
| 36 | +#include "iceberg/util/data_file_set.h" |
| 37 | +#include "iceberg/util/macros.h" |
| 38 | + |
| 39 | +namespace iceberg { |
| 40 | +namespace { |
| 41 | + |
| 42 | +struct CanonicalPartitionKey { |
| 43 | + int32_t spec_id; |
| 44 | + std::string values; |
| 45 | + |
| 46 | + auto operator<=>(const CanonicalPartitionKey&) const = default; |
| 47 | +}; |
| 48 | + |
| 49 | +struct Candidate { |
| 50 | + CompactionFile file; |
| 51 | + bool has_delete_pressure; |
| 52 | +}; |
| 53 | + |
| 54 | +struct PartitionCandidates { |
| 55 | + int32_t spec_id; |
| 56 | + PartitionValues partition; |
| 57 | + std::vector<Candidate> files; |
| 58 | +}; |
| 59 | + |
| 60 | +Status ValidateConfig(const CompactionPlannerConfig& config) { |
| 61 | + ICEBERG_PRECHECK(config.target_file_size_bytes > 0, |
| 62 | + "target_file_size_bytes must be greater than zero"); |
| 63 | + ICEBERG_PRECHECK(std::isfinite(config.min_file_size_ratio) && |
| 64 | + config.min_file_size_ratio >= 0 && config.min_file_size_ratio <= 1, |
| 65 | + "min_file_size_ratio must be finite and in [0, 1]"); |
| 66 | + ICEBERG_PRECHECK( |
| 67 | + std::isfinite(config.max_file_size_ratio) && config.max_file_size_ratio >= 1, |
| 68 | + "max_file_size_ratio must be finite and at least 1"); |
| 69 | + ICEBERG_PRECHECK(config.min_input_files > 0, |
| 70 | + "min_input_files must be greater than zero"); |
| 71 | + ICEBERG_PRECHECK(config.delete_file_threshold >= 0, |
| 72 | + "delete_file_threshold must not be negative"); |
| 73 | + ICEBERG_PRECHECK(std::isfinite(config.delete_ratio_threshold) && |
| 74 | + config.delete_ratio_threshold >= 0 && |
| 75 | + config.delete_ratio_threshold <= 1, |
| 76 | + "delete_ratio_threshold must be finite and in [0, 1]"); |
| 77 | + return {}; |
| 78 | +} |
| 79 | + |
| 80 | +Result<int64_t> CheckedAddNonNegative(int64_t lhs, int64_t rhs, |
| 81 | + std::string_view description) { |
| 82 | + ICEBERG_PRECHECK(lhs >= 0 && rhs >= 0, "{} must not be negative", description); |
| 83 | + ICEBERG_PRECHECK(lhs <= std::numeric_limits<int64_t>::max() - rhs, "{} overflow", |
| 84 | + description); |
| 85 | + return lhs + rhs; |
| 86 | +} |
| 87 | + |
| 88 | +void AppendFramed(std::string& output, std::string_view value) { |
| 89 | + output.append(std::to_string(value.size())); |
| 90 | + output.push_back(':'); |
| 91 | + output.append(value); |
| 92 | +} |
| 93 | + |
| 94 | +Result<CanonicalPartitionKey> MakePartitionKey(const DataFile& data_file) { |
| 95 | + ICEBERG_PRECHECK(data_file.partition_spec_id.has_value(), |
| 96 | + "Data file '{}' is missing partition_spec_id", data_file.file_path); |
| 97 | + |
| 98 | + std::string encoded; |
| 99 | + encoded.append(std::to_string(data_file.partition.num_fields())); |
| 100 | + encoded.push_back(':'); |
| 101 | + for (const auto& literal : data_file.partition.values()) { |
| 102 | + AppendFramed(encoded, literal.type()->ToString()); |
| 103 | + if (literal.IsNull()) { |
| 104 | + encoded.push_back('N'); |
| 105 | + continue; |
| 106 | + } |
| 107 | + |
| 108 | + encoded.push_back('V'); |
| 109 | + ICEBERG_ASSIGN_OR_RAISE(auto bytes, literal.Serialize()); |
| 110 | + std::string_view serialized; |
| 111 | + if (!bytes.empty()) { |
| 112 | + serialized = |
| 113 | + std::string_view(reinterpret_cast<const char*>(bytes.data()), bytes.size()); |
| 114 | + } |
| 115 | + AppendFramed(encoded, serialized); |
| 116 | + } |
| 117 | + return CanonicalPartitionKey{.spec_id = *data_file.partition_spec_id, |
| 118 | + .values = std::move(encoded)}; |
| 119 | +} |
| 120 | + |
| 121 | +Result<CompactionFile> BuildCompactionFile( |
| 122 | + const std::shared_ptr<FileScanTask>& scan_task) { |
| 123 | + CompactionFile result{.scan_task = scan_task}; |
| 124 | + const auto& data_file = scan_task->data_file(); |
| 125 | + ICEBERG_PRECHECK(data_file->record_count >= 0, |
| 126 | + "Data file '{}' has negative record count", data_file->file_path); |
| 127 | + ICEBERG_PRECHECK(data_file->file_size_in_bytes >= 0, |
| 128 | + "Data file '{}' has negative file size", data_file->file_path); |
| 129 | + |
| 130 | + DeleteFileSet applicable_deletes; |
| 131 | + for (const auto& delete_file : scan_task->delete_files()) { |
| 132 | + ICEBERG_PRECHECK(delete_file != nullptr, "Data file '{}' has a null delete file", |
| 133 | + data_file->file_path); |
| 134 | + if (delete_file->content != DataFile::Content::kPositionDeletes) { |
| 135 | + continue; |
| 136 | + } |
| 137 | + |
| 138 | + ICEBERG_ASSIGN_OR_RAISE(auto referenced_file, |
| 139 | + ContentFileUtil::ReferencedDataFile(*delete_file)); |
| 140 | + if (!referenced_file.has_value() || *referenced_file != data_file->file_path) { |
| 141 | + continue; |
| 142 | + } |
| 143 | + |
| 144 | + if (!applicable_deletes.insert(delete_file).second) { |
| 145 | + continue; |
| 146 | + } |
| 147 | + |
| 148 | + ICEBERG_PRECHECK(delete_file->record_count >= 0, |
| 149 | + "Delete file '{}' has negative record count", |
| 150 | + delete_file->file_path); |
| 151 | + if (delete_file->IsDeletionVector()) { |
| 152 | + ICEBERG_PRECHECK( |
| 153 | + delete_file->record_count <= data_file->record_count, |
| 154 | + "Deletion vector '{}' cardinality {} exceeds data file '{}' record count {}", |
| 155 | + delete_file->file_path, delete_file->record_count, data_file->file_path, |
| 156 | + data_file->record_count); |
| 157 | + } |
| 158 | + |
| 159 | + ICEBERG_ASSIGN_OR_RAISE(result.file_scoped_delete_count, |
| 160 | + CheckedAddNonNegative(result.file_scoped_delete_count, 1, |
| 161 | + "File-scoped delete-file count")); |
| 162 | + ICEBERG_ASSIGN_OR_RAISE(result.file_scoped_delete_record_count, |
| 163 | + CheckedAddNonNegative(result.file_scoped_delete_record_count, |
| 164 | + delete_file->record_count, |
| 165 | + "File-scoped delete-record count")); |
| 166 | + } |
| 167 | + |
| 168 | + result.file_scoped_delete_record_count = |
| 169 | + std::min(result.file_scoped_delete_record_count, data_file->record_count); |
| 170 | + return result; |
| 171 | +} |
| 172 | + |
| 173 | +bool CandidateLess(const Candidate& lhs, const Candidate& rhs) { |
| 174 | + const auto& lhs_data = *lhs.file.scan_task->data_file(); |
| 175 | + const auto& rhs_data = *rhs.file.scan_task->data_file(); |
| 176 | + return std::tie(lhs_data.file_path, lhs_data.file_size_in_bytes, lhs_data.record_count, |
| 177 | + lhs.file.file_scoped_delete_count, |
| 178 | + lhs.file.file_scoped_delete_record_count) < |
| 179 | + std::tie(rhs_data.file_path, rhs_data.file_size_in_bytes, rhs_data.record_count, |
| 180 | + rhs.file.file_scoped_delete_count, |
| 181 | + rhs.file.file_scoped_delete_record_count); |
| 182 | +} |
| 183 | + |
| 184 | +Result<void> AddToGroup(CompactionGroup& group, Candidate candidate, |
| 185 | + bool& has_delete_pressure) { |
| 186 | + const auto& data_file = *candidate.file.scan_task->data_file(); |
| 187 | + ICEBERG_ASSIGN_OR_RAISE( |
| 188 | + group.data_file_size_bytes, |
| 189 | + CheckedAddNonNegative(group.data_file_size_bytes, data_file.file_size_in_bytes, |
| 190 | + "Compaction group data-file size")); |
| 191 | + ICEBERG_ASSIGN_OR_RAISE( |
| 192 | + group.file_scoped_delete_record_count, |
| 193 | + CheckedAddNonNegative(group.file_scoped_delete_record_count, |
| 194 | + candidate.file.file_scoped_delete_record_count, |
| 195 | + "Compaction group delete-record count")); |
| 196 | + has_delete_pressure |= candidate.has_delete_pressure; |
| 197 | + group.files.push_back(std::move(candidate.file)); |
| 198 | + return {}; |
| 199 | +} |
| 200 | + |
| 201 | +void FinishGroup(CompactionGroup& group, bool has_delete_pressure, size_t min_input_files, |
| 202 | + CompactionPlan& plan) { |
| 203 | + if (!group.files.empty() && |
| 204 | + (has_delete_pressure || group.files.size() >= min_input_files)) { |
| 205 | + plan.groups.push_back(std::move(group)); |
| 206 | + } |
| 207 | +} |
| 208 | + |
| 209 | +} // namespace |
| 210 | + |
| 211 | +Result<CompactionPlan> CompactionPlanner::Plan( |
| 212 | + int64_t source_snapshot_id, std::span<const std::shared_ptr<FileScanTask>> scan_tasks, |
| 213 | + const CompactionPlannerConfig& config) { |
| 214 | + ICEBERG_RETURN_UNEXPECTED(ValidateConfig(config)); |
| 215 | + const long double min_file_size = |
| 216 | + config.target_file_size_bytes * config.min_file_size_ratio; |
| 217 | + const long double max_file_size = |
| 218 | + config.target_file_size_bytes * config.max_file_size_ratio; |
| 219 | + std::map<CanonicalPartitionKey, PartitionCandidates> partitions; |
| 220 | + |
| 221 | + for (const auto& scan_task : scan_tasks) { |
| 222 | + ICEBERG_PRECHECK(scan_task != nullptr, "File scan task must not be null"); |
| 223 | + const auto& data_file = *scan_task->data_file(); |
| 224 | + ICEBERG_ASSIGN_OR_RAISE(auto partition_key, MakePartitionKey(data_file)); |
| 225 | + ICEBERG_ASSIGN_OR_RAISE(auto file, BuildCompactionFile(scan_task)); |
| 226 | + |
| 227 | + const bool is_small = data_file.file_size_in_bytes < min_file_size; |
| 228 | + const bool is_oversized = data_file.file_size_in_bytes > max_file_size; |
| 229 | + const bool has_many_delete_files = |
| 230 | + config.delete_file_threshold > 0 && |
| 231 | + file.file_scoped_delete_count >= config.delete_file_threshold; |
| 232 | + const bool has_high_delete_ratio = |
| 233 | + config.delete_ratio_threshold > 0 && data_file.record_count > 0 && |
| 234 | + static_cast<double>(file.file_scoped_delete_record_count) / |
| 235 | + data_file.record_count >= |
| 236 | + config.delete_ratio_threshold; |
| 237 | + const bool has_delete_pressure = has_many_delete_files || has_high_delete_ratio; |
| 238 | + const bool should_compact = |
| 239 | + is_oversized ? has_delete_pressure : is_small || has_delete_pressure; |
| 240 | + if (!should_compact) { |
| 241 | + continue; |
| 242 | + } |
| 243 | + |
| 244 | + auto partition = |
| 245 | + partitions |
| 246 | + .try_emplace(std::move(partition_key), |
| 247 | + PartitionCandidates{.spec_id = *data_file.partition_spec_id, |
| 248 | + .partition = data_file.partition}) |
| 249 | + .first; |
| 250 | + partition->second.files.push_back( |
| 251 | + Candidate{.file = std::move(file), .has_delete_pressure = has_delete_pressure}); |
| 252 | + } |
| 253 | + |
| 254 | + CompactionPlan plan{.source_snapshot_id = source_snapshot_id}; |
| 255 | + for (auto& [_, partition] : partitions) { |
| 256 | + std::ranges::sort(partition.files, CandidateLess); |
| 257 | + |
| 258 | + CompactionGroup group{.partition_spec_id = partition.spec_id, |
| 259 | + .partition = partition.partition}; |
| 260 | + bool has_delete_pressure = false; |
| 261 | + for (auto& candidate : partition.files) { |
| 262 | + const int64_t file_size = candidate.file.scan_task->data_file()->file_size_in_bytes; |
| 263 | + if (!group.files.empty() && |
| 264 | + (group.data_file_size_bytes > config.target_file_size_bytes || |
| 265 | + file_size > config.target_file_size_bytes - group.data_file_size_bytes)) { |
| 266 | + FinishGroup(group, has_delete_pressure, config.min_input_files, plan); |
| 267 | + group = CompactionGroup{.partition_spec_id = partition.spec_id, |
| 268 | + .partition = partition.partition}; |
| 269 | + has_delete_pressure = false; |
| 270 | + } |
| 271 | + ICEBERG_RETURN_UNEXPECTED( |
| 272 | + AddToGroup(group, std::move(candidate), has_delete_pressure)); |
| 273 | + } |
| 274 | + FinishGroup(group, has_delete_pressure, config.min_input_files, plan); |
| 275 | + } |
| 276 | + return plan; |
| 277 | +} |
| 278 | + |
| 279 | +} // namespace iceberg |
0 commit comments