Skip to content

Commit cf28ec0

Browse files
manuzhangcodex
andcommitted
feat(inspect): add system metadata tables
Co-authored-by: Codex <codex@openai.com>
1 parent 6361791 commit cf28ec0

21 files changed

Lines changed: 2274 additions & 2 deletions

‎src/iceberg/CMakeLists.txt‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -53,9 +53,15 @@ set(ICEBERG_SOURCES
5353
file_reader.cc
5454
file_writer.cc
5555
geospatial.cc
56+
inspect/branches_table.cc
57+
inspect/files_table.cc
5658
inspect/history_table.cc
59+
inspect/manifests_table.cc
5760
inspect/metadata_table.cc
61+
inspect/metadata_table_util_internal.cc
62+
inspect/partitions_table.cc
5863
inspect/snapshots_table.cc
64+
inspect/tags_table.cc
5965
inheritable_metadata.cc
6066
json_serde.cc
6167
location_provider.cc
Lines changed: 108 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,108 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing,
13+
* software distributed under the License is distributed on an
14+
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
* KIND, either express or implied. See the License for the
16+
* specific language governing permissions and limitations
17+
* under the License.
18+
*/
19+
20+
#include "iceberg/inspect/branches_table.h"
21+
22+
#include <algorithm>
23+
#include <memory>
24+
#include <optional>
25+
#include <string>
26+
#include <utility>
27+
#include <vector>
28+
29+
#include "iceberg/arrow_row_builder_internal.h"
30+
#include "iceberg/inspect/metadata_table_stream_internal.h"
31+
#include "iceberg/schema.h"
32+
#include "iceberg/schema_field.h"
33+
#include "iceberg/snapshot.h"
34+
#include "iceberg/table.h"
35+
#include "iceberg/table_metadata.h"
36+
#include "iceberg/type.h"
37+
#include "iceberg/util/macros.h"
38+
39+
namespace iceberg {
40+
namespace {
41+
42+
struct BranchRow {
43+
std::string name;
44+
int64_t snapshot_id;
45+
std::optional<int64_t> max_ref_age_ms;
46+
std::optional<int32_t> min_snapshots_to_keep;
47+
std::optional<int64_t> max_snapshot_age_ms;
48+
};
49+
50+
Status AppendOptional(ArrowArray* array, const auto& value) {
51+
if (!value.has_value()) {
52+
return AppendNull(array);
53+
}
54+
return AppendInt(array, static_cast<int64_t>(*value));
55+
}
56+
57+
Status AppendBranch(ArrowRowBuilder& builder, const BranchRow& branch) {
58+
ICEBERG_RETURN_UNEXPECTED(AppendString(builder.column(0), branch.name));
59+
ICEBERG_RETURN_UNEXPECTED(AppendInt(builder.column(1), branch.snapshot_id));
60+
ICEBERG_RETURN_UNEXPECTED(AppendOptional(builder.column(2), branch.max_ref_age_ms));
61+
ICEBERG_RETURN_UNEXPECTED(
62+
AppendOptional(builder.column(3), branch.min_snapshots_to_keep));
63+
ICEBERG_RETURN_UNEXPECTED(
64+
AppendOptional(builder.column(4), branch.max_snapshot_age_ms));
65+
return builder.FinishRow();
66+
}
67+
68+
} // namespace
69+
70+
BranchesTable::BranchesTable(std::shared_ptr<Table> table)
71+
: MetadataTable(std::move(table)) {}
72+
73+
BranchesTable::~BranchesTable() = default;
74+
75+
const std::shared_ptr<Schema>& BranchesTable::schema() const {
76+
static const auto schema = std::make_shared<Schema>(std::vector<SchemaField>{
77+
SchemaField::MakeRequired(1, "name", string()),
78+
SchemaField::MakeRequired(2, "snapshot_id", int64()),
79+
SchemaField::MakeOptional(3, "max_reference_age_in_ms", int64()),
80+
SchemaField::MakeOptional(4, "min_snapshots_to_keep", int32()),
81+
SchemaField::MakeOptional(5, "max_snapshot_age_in_ms", int64()),
82+
});
83+
return schema;
84+
}
85+
86+
Result<std::unique_ptr<BranchesTable>> BranchesTable::Make(std::shared_ptr<Table> table) {
87+
ICEBERG_PRECHECK(table != nullptr, "Table cannot be null");
88+
return std::unique_ptr<BranchesTable>(new BranchesTable(std::move(table)));
89+
}
90+
91+
Result<ArrowArrayStream> BranchesTable::Scan() {
92+
std::vector<BranchRow> rows;
93+
for (const auto& [name, ref] : source_table()->metadata()->refs) {
94+
if (ref == nullptr || ref->type() != SnapshotRefType::kBranch) {
95+
continue;
96+
}
97+
const auto& retention = std::get<SnapshotRef::Branch>(ref->retention);
98+
rows.push_back(BranchRow{.name = name,
99+
.snapshot_id = ref->snapshot_id,
100+
.max_ref_age_ms = retention.max_ref_age_ms,
101+
.min_snapshots_to_keep = retention.min_snapshots_to_keep,
102+
.max_snapshot_age_ms = retention.max_snapshot_age_ms});
103+
}
104+
std::ranges::sort(rows, {}, &BranchRow::name);
105+
return internal::MakeMetadataTableStream(*schema(), std::move(rows), AppendBranch);
106+
}
107+
108+
} // namespace iceberg
Lines changed: 51 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,51 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing,
13+
* software distributed under the License is distributed on an
14+
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
* KIND, either express or implied. See the License for the
16+
* specific language governing permissions and limitations
17+
* under the License.
18+
*/
19+
20+
#pragma once
21+
22+
/// \file iceberg/inspect/branches_table.h
23+
/// \brief Define the branches metadata table.
24+
25+
#include <memory>
26+
27+
#include "iceberg/iceberg_export.h"
28+
#include "iceberg/inspect/metadata_table.h"
29+
#include "iceberg/result.h"
30+
#include "iceberg/type_fwd.h"
31+
32+
namespace iceberg {
33+
34+
/// \brief Metadata table containing the table's branch snapshot references.
35+
class ICEBERG_EXPORT BranchesTable : public MetadataTable {
36+
public:
37+
static Result<std::unique_ptr<BranchesTable>> Make(std::shared_ptr<Table> table);
38+
39+
~BranchesTable() override;
40+
41+
Kind kind() const noexcept override { return Kind::kBranches; }
42+
43+
const std::shared_ptr<Schema>& schema() const override;
44+
45+
Result<ArrowArrayStream> Scan() override;
46+
47+
private:
48+
explicit BranchesTable(std::shared_ptr<Table> table);
49+
};
50+
51+
} // namespace iceberg

‎src/iceberg/inspect/files_table.cc‎

Lines changed: 75 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,75 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing,
13+
* software distributed under the License is distributed on an
14+
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
* KIND, either express or implied. See the License for the
16+
* specific language governing permissions and limitations
17+
* under the License.
18+
*/
19+
20+
#include "iceberg/inspect/files_table.h"
21+
22+
#include <memory>
23+
#include <utility>
24+
25+
#include "iceberg/inspect/metadata_table_stream_internal.h"
26+
#include "iceberg/inspect/metadata_table_util_internal.h"
27+
#include "iceberg/schema.h"
28+
#include "iceberg/table.h"
29+
#include "iceberg/type.h"
30+
#include "iceberg/util/macros.h"
31+
32+
namespace iceberg {
33+
34+
FilesTable::FilesTable(std::shared_ptr<Table> table, std::shared_ptr<Schema> schema,
35+
std::shared_ptr<Schema> table_schema,
36+
std::shared_ptr<StructType> partition_type)
37+
: TimeTravelMetadataTable(std::move(table)),
38+
schema_(std::move(schema)),
39+
table_schema_(std::move(table_schema)),
40+
partition_type_(std::move(partition_type)) {}
41+
42+
FilesTable::~FilesTable() = default;
43+
44+
const std::shared_ptr<Schema>& FilesTable::schema() const { return schema_; }
45+
46+
Result<std::unique_ptr<FilesTable>> FilesTable::Make(std::shared_ptr<Table> table) {
47+
ICEBERG_PRECHECK(table != nullptr, "Table cannot be null");
48+
ICEBERG_ASSIGN_OR_RAISE(auto table_schema, table->schema());
49+
ICEBERG_ASSIGN_OR_RAISE(auto partition_type, internal::UnifiedPartitionType(*table));
50+
ICEBERG_ASSIGN_OR_RAISE(auto schema,
51+
internal::FilesTableSchema(*table_schema, partition_type));
52+
return std::unique_ptr<FilesTable>(new FilesTable(std::move(table), std::move(schema),
53+
std::move(table_schema),
54+
std::move(partition_type)));
55+
}
56+
57+
Result<ArrowArrayStream> FilesTable::ScanSnapshot(
58+
const SnapshotSelection& snapshot_selection) {
59+
ICEBERG_ASSIGN_OR_RAISE(auto snapshot, internal::ResolveMetadataTableSnapshot(
60+
*source_table(), snapshot_selection));
61+
ICEBERG_ASSIGN_OR_RAISE(auto files, internal::LoadLiveFiles(*source_table(), snapshot));
62+
auto schema = schema_;
63+
auto table_schema = table_schema_;
64+
auto partition_type = partition_type_;
65+
return internal::MakeMetadataTableStream(
66+
*schema_, std::move(files),
67+
[schema = std::move(schema), table_schema = std::move(table_schema),
68+
partition_type = std::move(partition_type)](ArrowRowBuilder& builder,
69+
const internal::LiveFile& file) {
70+
return internal::AppendDataFile(builder, *schema, *table_schema, *partition_type,
71+
file);
72+
});
73+
}
74+
75+
} // namespace iceberg

‎src/iceberg/inspect/files_table.h‎

Lines changed: 59 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,59 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing,
13+
* software distributed under the License is distributed on an
14+
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
* KIND, either express or implied. See the License for the
16+
* specific language governing permissions and limitations
17+
* under the License.
18+
*/
19+
20+
#pragma once
21+
22+
/// \file iceberg/inspect/files_table.h
23+
/// \brief Define the files metadata table.
24+
25+
#include <memory>
26+
27+
#include "iceberg/iceberg_export.h"
28+
#include "iceberg/inspect/metadata_table.h"
29+
#include "iceberg/result.h"
30+
#include "iceberg/type_fwd.h"
31+
32+
namespace iceberg {
33+
34+
/// \brief Metadata table containing live data and delete files in a snapshot.
35+
class ICEBERG_EXPORT FilesTable : public TimeTravelMetadataTable {
36+
public:
37+
static Result<std::unique_ptr<FilesTable>> Make(std::shared_ptr<Table> table);
38+
39+
~FilesTable() override;
40+
41+
Kind kind() const noexcept override { return Kind::kFiles; }
42+
43+
const std::shared_ptr<Schema>& schema() const override;
44+
45+
protected:
46+
Result<ArrowArrayStream> ScanSnapshot(
47+
const SnapshotSelection& snapshot_selection) override;
48+
49+
private:
50+
FilesTable(std::shared_ptr<Table> table, std::shared_ptr<Schema> schema,
51+
std::shared_ptr<Schema> table_schema,
52+
std::shared_ptr<StructType> partition_type);
53+
54+
std::shared_ptr<Schema> schema_;
55+
std::shared_ptr<Schema> table_schema_;
56+
std::shared_ptr<StructType> partition_type_;
57+
};
58+
59+
} // namespace iceberg

0 commit comments

Comments
 (0)