Write the list layout when the enabled editions include it (#10563)

## Summary

The default writer now writes the list layout whenever the session's
enabled editions include `vortex.list`, so the editions are the only
switch. `with_list_layout()` is removed, with no opt-out in its place.
No default edition includes `vortex.list`, so default writes are
unchanged.

## Changes

- `WriteStrategyBuilder::from_session` turns the list layout on when the
enabled editions include `vortex.list`.
- Removes `WriteStrategyBuilder::with_list_layout()`.
- vortex-file's test session enables every registered layout, so its
list tests now write the list layout; the flat/list switch in
`write_read_roundtrip_with_layout` is removed.

## API Changes

- Removed `WriteStrategyBuilder::with_list_layout()` (added in #10529,
unreleased).

---------

Signed-off-by: Matt Katz <mhkatz97@gmail.com>

Author: mhk197

vortex-file/src/strategy.rs

@@ -10,15 +10,19 @@ use vortex_array::dtype::FieldPath;
 use vortex_btrblocks::BtrBlocksCompressorBuilder;
 use vortex_btrblocks::SchemeExt;
 use vortex_btrblocks::schemes::integer::IntDictScheme;
+use vortex_edition::ComponentKind;
+use vortex_edition::EditionSessionExt;
 use vortex_error::VortexExpect;
 use vortex_layout::LayoutStrategy;
+use vortex_layout::VTable;
 use vortex_layout::layouts::buffered::BufferedStrategy;
 use vortex_layout::layouts::chunked::writer::ChunkedLayoutStrategy;
 use vortex_layout::layouts::collect::CollectStrategy;
 use vortex_layout::layouts::compressed::CompressingStrategy;
 use vortex_layout::layouts::compressed::CompressorPlugin;
 use vortex_layout::layouts::dict::writer::DictStrategy;
 use vortex_layout::layouts::flat::writer::FlatLayoutStrategy;
+use vortex_layout::layouts::list::List;
 use vortex_layout::layouts::list::writer::ListLayoutStrategy;
 use vortex_layout::layouts::repartition::RepartitionStrategy;
 use vortex_layout::layouts::repartition::RepartitionWriterOptions;
@@ -77,7 +81,9 @@ impl WriteStrategyBuilder {
             field_writers: HashMap::new(),
             flat_strategy: None,
             probe_compressor: None,
-            use_list_layout: false,
+            use_list_layout: session
+                .enabled_component_ids(ComponentKind::Layout)
+                .contains(&List.id()),
         }
     }
 
@@ -99,19 +105,6 @@ impl WriteStrategyBuilder {
         self
     }
 
-    /// Enable writing list fields with [`ListLayoutStrategy`].
-    ///
-    /// **Note**: this is an unstable and experimental layout that is expected to change.
-    /// Using it may lead to unreadable files in the future.
-    ///
-    /// Writing fails unless the session's enabled editions include the `vortex.list` layout.
-    ///
-    /// [`ListLayoutStrategy`]: vortex_layout::layouts::list::writer::ListLayoutStrategy
-    pub fn with_list_layout(mut self) -> Self {
-        self.use_list_layout = true;
-        self
-    }
-
     /// Override the write layout for a specific field somewhere in the nested schema tree.
     ///
     /// The field path is matched after the root struct is split into columns. This is useful when a

vortex-file/src/tests.rs

@@ -2010,32 +2010,14 @@ async fn test_writer_with_complex_types() -> VortexResult<()> {
     Ok(())
 }
 
-/// Write `array` with list decomposition forced on (through the full compress/zone pipeline) and
-/// read the whole thing back.
+/// Write `array` through the full compress/zone pipeline and read the whole thing back. `SESSION`
+/// enables the `vortex.list` layout, so lists are decomposed.
 async fn write_read_roundtrip(array: ArrayRef) -> VortexResult<ArrayRef> {
-    write_read_roundtrip_with_layout(array, true).await
-}
-
-async fn write_read_roundtrip_with_layout(
-    array: ArrayRef,
-    use_list_layout: bool,
-) -> VortexResult<ArrayRef> {
-    let strategy = crate::strategy::WriteStrategyBuilder::from_session(&SESSION)
-        .with_list_layout()
-        .build();
     let mut buf = ByteBufferMut::empty();
-    if use_list_layout {
-        SESSION
-            .write_options()
-            .with_strategy(strategy)
-            .write(&mut buf, array.to_array_stream())
-            .await?;
-    } else {
-        SESSION
-            .write_options()
-            .write(&mut buf, array.to_array_stream())
-            .await?;
-    }
+    SESSION
+        .write_options()
+        .write(&mut buf, array.to_array_stream())
+        .await?;
     SESSION
         .open_options()
         .open_buffer(buf)?
@@ -2110,30 +2092,28 @@ fn map_array_from_rows(rows: &[MapRowFixture<'_>], keys_sorted: bool) -> VortexR
     Ok(builder.finish_into_map().into_array())
 }
 
-/// A struct containing a Map column crosses both the default flat writer and the list layout
-/// strategy without changing map nullability, empty rows, duplicate keys, or scalar values.
+/// A struct containing a Map column crosses the list layout strategy without changing map
+/// nullability, empty rows, duplicate keys, or scalar values.
 #[tokio::test]
 #[cfg_attr(miri, ignore)]
 async fn struct_with_map_column_roundtrip() -> VortexResult<()> {
-    for use_list_layout in [false, true] {
-        let maps = map_array_from_rows(
-            &[
-                Some(vec![(1, Some("one")), (2, None)]),
-                None,
-                Some(vec![]),
-                Some(vec![(1, Some("dup-old")), (1, Some("dup-new"))]),
-            ],
-            false,
-        )?;
-        let st = StructArray::from_fields(&[
-            ("id", buffer![10i32, 20, 30, 40].into_array()),
-            ("attrs", maps),
-        ])?
-        .into_array();
+    let maps = map_array_from_rows(
+        &[
+            Some(vec![(1, Some("one")), (2, None)]),
+            None,
+            Some(vec![]),
+            Some(vec![(1, Some("dup-old")), (1, Some("dup-new"))]),
+        ],
+        false,
+    )?;
+    let st = StructArray::from_fields(&[
+        ("id", buffer![10i32, 20, 30, 40].into_array()),
+        ("attrs", maps),
+    ])?
+    .into_array();
 
-        let result = write_read_roundtrip_with_layout(st.clone(), use_list_layout).await?;
-        assert_arrays_eq!(result, st, &mut SESSION.create_execution_ctx());
-    }
+    let result = write_read_roundtrip(st.clone()).await?;
+    assert_arrays_eq!(result, st, &mut SESSION.create_execution_ctx());
 
     Ok(())
 }

vortex/src/editions/tests.rs

@@ -440,24 +440,6 @@ async fn write_with(session: &VortexSession, array: ArrayRef) -> VortexResult<By
     Ok(buffer)
 }
 
-/// Write `array` with `session`, opting in to the list layout.
-async fn write_list_layout(
-    session: &VortexSession,
-    array: ArrayRef,
-) -> VortexResult<ByteBufferMut> {
-    let mut buffer = ByteBufferMut::empty();
-    session
-        .write_options()
-        .with_strategy(
-            WriteStrategyBuilder::from_session(session)
-                .with_list_layout()
-                .build(),
-        )
-        .write(&mut buffer, array.to_array_stream())
-        .await?;
-    Ok(buffer)
-}
-
 /// The layout encodings a written file actually contains, depth first.
 fn written_layout_ids(session: &VortexSession, buffer: ByteBufferMut) -> VortexResult<Vec<Id>> {
     let file = session.open_options().open_buffer(buffer)?;
@@ -562,10 +544,9 @@ async fn writer_restricts_layouts_to_the_enabled_editions() -> VortexResult<()>
     Ok(())
 }
 
-/// The list layout is opt-in: the default writer never emits it, and opting in writes it only when
-/// the enabled editions permit it.
+/// The writer emits the list layout exactly when the enabled editions include it.
 #[tokio::test]
-async fn list_layout_is_opt_in_and_edition_gated() -> VortexResult<()> {
+async fn list_layout_follows_enabled_editions() -> VortexResult<()> {
     use crate::VortexSessionDefault;
 
     let list = ListArray::try_new(
@@ -576,15 +557,10 @@ async fn list_layout_is_opt_in_and_edition_gated() -> VortexResult<()> {
     let array = StructArray::from_fields(&[("l", list.into_array())])?.into_array();
     let list_layout = Id::from("vortex.list");
 
-    // The default core edition does not include the list layout, so opting in fails the write.
+    // The default core edition does not include the list layout, so lists stay flat.
     let session = VortexSession::default();
-    let error = write_list_layout(&session, array.clone())
-        .await
-        .expect_err("the list layout is not in the default editions");
-    assert!(
-        error.to_string().contains("not permitted by ctx"),
-        "unexpected error: {error}"
-    );
+    let written = written_layout_ids(&session, write_with(&session, array.clone()).await?)?;
+    assert!(!written.contains(&list_layout), "{written:?}");
 
     let session = VortexSession::default();
     session.editions().declare_inclusion(EditionInclusion::new(
@@ -594,12 +570,8 @@ async fn list_layout_is_opt_in_and_edition_gated() -> VortexResult<()> {
     ))?;
     session.enable_edition(DEFAULT_PREVIEW_EDITION)?;
 
-    // Permitted but not requested: the default writer keeps lists flat.
-    let written = written_layout_ids(&session, write_with(&session, array.clone()).await?)?;
-    assert!(!written.contains(&list_layout), "{written:?}");
-
-    // Permitted and requested.
-    let written = written_layout_ids(&session, write_list_layout(&session, array).await?)?;
+    // Permitted: the default writer emits it.
+    let written = written_layout_ids(&session, write_with(&session, array).await?)?;
     assert!(written.contains(&list_layout), "{written:?}");
     Ok(())
 }