fix(inspect): align metadata tables with Java

Author: manuzhang

src/iceberg/CMakeLists.txt

@@ -53,15 +53,14 @@ set(ICEBERG_SOURCES
     file_reader.cc
     file_writer.cc
     geospatial.cc
-    inspect/branches_table.cc
     inspect/files_table.cc
     inspect/history_table.cc
     inspect/manifests_table.cc
     inspect/metadata_table.cc
     inspect/metadata_table_util_internal.cc
     inspect/partitions_table.cc
+    inspect/refs_table.cc
     inspect/snapshots_table.cc
-    inspect/tags_table.cc
     inheritable_metadata.cc
     json_serde.cc
     location_provider.cc

src/iceberg/arrow_row_builder.cc

@@ -28,6 +28,7 @@
 #include "iceberg/nanoarrow_status_internal.h"
 #include "iceberg/schema.h"
 #include "iceberg/schema_internal.h"
+#include "iceberg/util/decimal.h"
 
 namespace iceberg {
 
@@ -123,6 +124,15 @@ Status AppendDouble(ArrowArray* array, double value) {
   return {};
 }
 
+Status AppendDecimal(ArrowArray* array, const Decimal& value) {
+  // Append the unscaled integer; the array schema supplies precision and scale.
+  ArrowDecimal decimal;
+  ArrowDecimalInit(&decimal, Decimal::kBitWidth, Decimal::kMaxPrecision, 0);
+  ArrowDecimalSetBytes(&decimal, value.native_endian_bytes());
+  ICEBERG_NANOARROW_RETURN_UNEXPECTED(ArrowArrayAppendDecimal(array, &decimal));
+  return {};
+}
+
 Status AppendString(ArrowArray* array, std::string_view value) {
   ArrowStringView view(value.data(), static_cast<int64_t>(value.size()));
   ICEBERG_NANOARROW_RETURN_UNEXPECTED(ArrowArrayAppendString(array, view));

src/iceberg/arrow_row_builder_internal.h

@@ -125,6 +125,9 @@ ICEBERG_EXPORT Status AppendUInt(ArrowArray* array, uint64_t value);
 /// \brief Append a double value to a nanoarrow array builder.
 ICEBERG_EXPORT Status AppendDouble(ArrowArray* array, double value);
 
+/// \brief Append an unscaled decimal value to a decimal128 array builder.
+ICEBERG_EXPORT Status AppendDecimal(ArrowArray* array, const Decimal& value);
+
 /// \brief Append a string value to a nanoarrow array builder.
 ICEBERG_EXPORT Status AppendString(ArrowArray* array, std::string_view value);
 

src/iceberg/inspect/branches_table.h

@@ -58,7 +58,7 @@ Result<ArrowArrayStream> FilesTable::ScanSnapshot(
     const SnapshotSelection& snapshot_selection) {
   ICEBERG_ASSIGN_OR_RAISE(auto snapshot, internal::ResolveMetadataTableSnapshot(
                                              *source_table(), snapshot_selection));
-  ICEBERG_ASSIGN_OR_RAISE(auto files, internal::LoadLiveFiles(*source_table(), snapshot));
+  ICEBERG_ASSIGN_OR_RAISE(auto files, internal::LiveFiles(*source_table(), snapshot));
   auto schema = schema_;
   auto table_schema = table_schema_;
   auto partition_type = partition_type_;

src/iceberg/inspect/files_table.cc

@@ -90,15 +90,15 @@ Status AppendPartitionSummaries(ArrowArray* array, const ManifestRow& row,
           HumanReadableBound(*row.spec, *partition_type, index, *summary.lower_bound));
       ICEBERG_RETURN_UNEXPECTED(AppendString(entries->children[2], lower));
     } else {
-      ICEBERG_RETURN_UNEXPECTED(AppendNull(entries->children[2]));
+      ICEBERG_RETURN_UNEXPECTED(AppendString(entries->children[2], "null"));
     }
     if (summary.upper_bound.has_value()) {
       ICEBERG_ASSIGN_OR_RAISE(
           auto upper,
           HumanReadableBound(*row.spec, *partition_type, index, *summary.upper_bound));
       ICEBERG_RETURN_UNEXPECTED(AppendString(entries->children[3], upper));
     } else {
-      ICEBERG_RETURN_UNEXPECTED(AppendNull(entries->children[3]));
+      ICEBERG_RETURN_UNEXPECTED(AppendString(entries->children[3], "null"));
     }
     ICEBERG_NANOARROW_RETURN_UNEXPECTED(ArrowArrayFinishElement(entries));
   }

src/iceberg/inspect/manifests_table.cc

@@ -17,14 +17,13 @@
 
 install_headers(
     [
-        'branches_table.h',
         'files_table.h',
         'history_table.h',
         'manifests_table.h',
         'metadata_table.h',
         'partitions_table.h',
+        'refs_table.h',
         'snapshots_table.h',
-        'tags_table.h',
     ],
     subdir: 'iceberg/inspect',
 )

src/iceberg/inspect/meson.build

@@ -43,8 +43,7 @@ class ICEBERG_EXPORT MetadataTable {
   enum class Kind {
     kSnapshots,
     kHistory,
-    kBranches,
-    kTags,
+    kRefs,
     kFiles,
     kPartitions,
     kManifests,
@@ -98,9 +97,10 @@ struct SnapshotSelection {
   /// std::monostate selects the current snapshot.
   std::variant<std::monostate, int64_t, TimePointMs> snapshot;
 
-  /// \brief Resolve the snapshot relative to this branch or tag.
+  /// \brief Select the head of this branch or tag.
   ///
-  /// An empty string uses the main branch.
+  /// An empty string or "main" uses the main branch. A non-main reference cannot
+  /// be combined with a snapshot ID or timestamp.
   std::string ref_name;
 };
 

src/iceberg/inspect/metadata_table.h

@@ -34,19 +34,19 @@
 #include "iceberg/nanoarrow_status_internal.h"
 #include "iceberg/schema.h"
 #include "iceberg/schema_internal.h"
+#include "iceberg/util/iterator.h"
 #include "iceberg/util/macros.h"
 
 namespace iceberg::internal {
 
-/// \brief Arrow stream backed by a fixed set of metadata-table rows.
+/// \brief Arrow stream backed by a metadata-table row iterator.
 template <typename Row>
 class MetadataTableRowsStream {
  public:
   using AppendRow = std::function<Status(ArrowRowBuilder&, const Row&)>;
 
-  static Result<std::unique_ptr<MetadataTableRowsStream>> Make(const Schema& schema,
-                                                               std::vector<Row> rows,
-                                                               AppendRow append_row) {
+  static Result<std::unique_ptr<MetadataTableRowsStream>> Make(
+      const Schema& schema, std::unique_ptr<Iterator<Row>> rows, AppendRow append_row) {
     ArrowSchema arrow_schema{};
     ICEBERG_RETURN_UNEXPECTED(ToArrowSchema(schema, &arrow_schema));
     return std::unique_ptr<MetadataTableRowsStream>(new MetadataTableRowsStream(
@@ -59,7 +59,7 @@ class MetadataTableRowsStream {
   }
 
   Status Close() {
-    rows_.clear();
+    rows_.reset();
     append_row_ = nullptr;
     if (arrow_schema_.release != nullptr) {
       ArrowSchemaRelease(&arrow_schema_);
@@ -70,13 +70,16 @@ class MetadataTableRowsStream {
   Result<std::optional<ArrowArray>> Next() {
     ICEBERG_PRECHECK(arrow_schema_.release != nullptr,
                      "Cannot read from a closed metadata table stream");
-    if (next_row_ == rows_.size()) {
-      return std::nullopt;
-    }
-
     ICEBERG_ASSIGN_OR_RAISE(auto builder, ArrowRowBuilder::Make(&arrow_schema_));
-    while (next_row_ < rows_.size() && builder.num_rows() < MetadataTable::kBatchSize) {
-      ICEBERG_RETURN_UNEXPECTED(append_row_(builder, rows_[next_row_++]));
+    while (builder.num_rows() < MetadataTable::kBatchSize) {
+      ICEBERG_ASSIGN_OR_RAISE(auto row, rows_->Next());
+      if (!row.has_value()) {
+        break;
+      }
+      ICEBERG_RETURN_UNEXPECTED(append_row_(builder, *row));
+    }
+    if (builder.num_rows() == 0) {
+      return std::nullopt;
     }
 
     ICEBERG_ASSIGN_OR_RAISE(auto array, std::move(builder).Finish());
@@ -93,21 +96,20 @@ class MetadataTableRowsStream {
   }
 
  private:
-  MetadataTableRowsStream(std::vector<Row> rows, AppendRow append_row,
+  MetadataTableRowsStream(std::unique_ptr<Iterator<Row>> rows, AppendRow append_row,
                           ArrowSchema arrow_schema)
       : rows_(std::move(rows)),
         append_row_(std::move(append_row)),
         arrow_schema_(std::move(arrow_schema)) {}
 
-  std::vector<Row> rows_;
+  std::unique_ptr<Iterator<Row>> rows_;
   AppendRow append_row_;
   ArrowSchema arrow_schema_{};
-  size_t next_row_ = 0;
 };
 
 template <typename Row, typename AppendRow>
 Result<ArrowArrayStream> MakeMetadataTableStream(const Schema& schema,
-                                                 std::vector<Row> rows,
+                                                 std::unique_ptr<Iterator<Row>> rows,
                                                  AppendRow append_row) {
   ICEBERG_ASSIGN_OR_RAISE(
       auto stream,
@@ -117,4 +119,31 @@ Result<ArrowArrayStream> MakeMetadataTableStream(const Schema& schema,
   return MakeArrowArrayStream(std::move(stream));
 }
 
+template <typename Row>
+class MetadataTableVectorIterator : public Iterator<Row> {
+ public:
+  explicit MetadataTableVectorIterator(std::vector<Row> rows) : rows_(std::move(rows)) {}
+
+ protected:
+  Result<std::optional<Row>> NextImpl() override {
+    if (next_row_ == rows_.size()) {
+      return std::nullopt;
+    }
+    return std::move(rows_[next_row_++]);
+  }
+
+ private:
+  std::vector<Row> rows_;
+  size_t next_row_ = 0;
+};
+
+template <typename Row, typename AppendRow>
+Result<ArrowArrayStream> MakeMetadataTableStream(const Schema& schema,
+                                                 std::vector<Row> rows,
+                                                 AppendRow append_row) {
+  std::unique_ptr<Iterator<Row>> iterator =
+      std::make_unique<MetadataTableVectorIterator<Row>>(std::move(rows));
+  return MakeMetadataTableStream(schema, std::move(iterator), std::move(append_row));
+}
+
 }  // namespace iceberg::internal