Use mimalloc with purging disabled in each e2e benchmark binary (#10490)

The end-to-end benchmark binaries (SQL via DataFusion and DuckDB, random
access, compression, string, Lance) only ran with mimalloc implicitly,
through a `#[global_allocator]` declared in the `vortex-bench` library.
This makes the choice explicit in each benchmark binary, and stops
mimalloc from purging freed memory back to the OS during a run: mimalloc
v3 purges 1s after a free, and that work lands in the middle of measured
queries.

---------

Signed-off-by: Joe Isaacs <joe.isaacs@live.co.uk>

Author: joseph-isaacs

Cargo.lock

@@ -191,6 +191,7 @@ lending-iterator = "0.1.7"
 libfuzzer-sys = "0.4"
 libloading = "0.8"
 liblzma = "0.4"
+libmimalloc-sys = { version = "0.1.49", features = ["extended"] }
 log = { version = "0.4.21" }
 memchr = "2.8.1"
 memmap2 = "0.9.5"

Cargo.toml

@@ -25,6 +25,7 @@ futures = { workspace = true }
 indicatif = { workspace = true }
 itertools = { workspace = true }
 lance-bench = { path = "../lance-bench", optional = true }
+mimalloc = { workspace = true }
 parquet = { workspace = true }
 regex = { workspace = true }
 serde = { workspace = true, features = ["derive"] }

benchmarks/compress-bench/Cargo.toml

@@ -20,6 +20,7 @@ use compress_bench::vortex::VortexCompressor;
 use futures::FutureExt;
 use indicatif::ProgressBar;
 use itertools::Itertools;
+use mimalloc::MiMalloc;
 use regex::Regex;
 use vortex::utils::aliases::hash_map::HashMap;
 use vortex_bench::Engine;
@@ -126,7 +127,11 @@ struct Args {
     log_format: LogFormat,
 }
 
+#[global_allocator]
+static GLOBAL: MiMalloc = MiMalloc;
+
 fn main() -> anyhow::Result<()> {
+    vortex_bench::configure_allocator();
     let args = Args::parse();
 
     setup_logging_and_tracing_with_format(args.verbose, args.tracing, args.log_format)?;

benchmarks/compress-bench/src/main.rs

@@ -35,6 +35,7 @@ datafusion-common = { workspace = true }
 datafusion-physical-plan = { workspace = true }
 futures.workspace = true
 itertools.workspace = true
+mimalloc = { workspace = true }
 object_store = { workspace = true, features = ["aws", "gcp"] }
 opentelemetry.workspace = true
 opentelemetry-otlp.workspace = true

benchmarks/datafusion-bench/Cargo.toml

@@ -24,6 +24,7 @@ use datafusion_bench::tracer::set_labels;
 use datafusion_common::TableReference;
 use datafusion_physical_plan::ExecutionPlan;
 use datafusion_physical_plan::collect;
+use mimalloc::MiMalloc;
 use parking_lot::Mutex;
 use vortex::file::multi::MultiFileDataSource;
 use vortex::io::filesystem::FileSystemRef;
@@ -123,8 +124,12 @@ struct Args {
     options: Vec<Opt>,
 }
 
+#[global_allocator]
+static GLOBAL: MiMalloc = MiMalloc;
+
 #[tokio::main]
 async fn main() -> anyhow::Result<()> {
+    vortex_bench::configure_allocator();
     let args = Args::parse();
     let opts = Opts::from(args.options);
 

benchmarks/datafusion-bench/src/main.rs

@@ -23,6 +23,7 @@ test = false
 [dependencies]
 anyhow = { workspace = true }
 clap = { workspace = true, features = ["derive"] }
+mimalloc = { workspace = true }
 similar = { workspace = true }
 tokio = { workspace = true }
 tracing = { workspace = true }

benchmarks/duckdb-bench/Cargo.toml

@@ -8,6 +8,7 @@ use std::path::PathBuf;
 use clap::Parser;
 use clap::value_parser;
 use duckdb_bench::DuckClient;
+use mimalloc::MiMalloc;
 use tokio::runtime::Runtime;
 use vortex::metrics::tracing::set_global_labels;
 use vortex_bench::BenchmarkArg;
@@ -103,7 +104,11 @@ struct Args {
     reuse: bool,
 }
 
+#[global_allocator]
+static GLOBAL: MiMalloc = MiMalloc;
+
 fn main() -> anyhow::Result<()> {
+    vortex_bench::configure_allocator();
     let args = Args::parse();
     let opts = Opts::from(args.options);
 

benchmarks/duckdb-bench/src/main.rs

@@ -24,6 +24,7 @@ async-trait = { workspace = true }
 clap = { workspace = true, features = ["derive"] }
 futures = { workspace = true }
 glob = { workspace = true }
+mimalloc = { workspace = true }
 parquet = { version = "58" }
 tempfile = { workspace = true }
 tokio = { workspace = true, features = ["full"] }

benchmarks/lance-bench/Cargo.toml

@@ -14,6 +14,7 @@ use lance::deps::datafusion::arrow::util::pretty::pretty_format_batches;
 use lance::deps::datafusion::physical_plan::ExecutionPlan;
 use lance::deps::datafusion::prelude::SessionContext;
 use lance_bench::convert::convert_parquet_to_lance;
+use mimalloc::MiMalloc;
 use tracing::info;
 use vortex_bench::Benchmark;
 use vortex_bench::BenchmarkArg;
@@ -84,8 +85,12 @@ struct Args {
     options: Vec<Opt>,
 }
 
+#[global_allocator]
+static GLOBAL: MiMalloc = MiMalloc;
+
 #[tokio::main]
 async fn main() -> anyhow::Result<()> {
+    vortex_bench::configure_allocator();
     let args = Args::parse();
     let opts = Opts::from(args.options);