Merge branch 'main' into fix/binary-map-initial-map-size

Author: yashrb24

.github/workflows/rust.yml

@@ -305,7 +305,7 @@ jobs:
             --lib \
             --tests \
             --bins \
-            --features serde,avro,json,backtrace,integration-tests,parquet_encryption
+            --features serde,avro,json,backtrace,integration-tests,parquet_encryption,substrait
       - name: Verify Working Directory Clean
         run: git diff --exit-code
         # Check no temporary directories created during test.
@@ -473,7 +473,7 @@ jobs:
           export RUST_MIN_STACK=20971520
           export TPCH_DATA=`realpath datafusion/sqllogictest/test_files/tpch/data`
           cargo test plan_q --package datafusion-benchmarks --profile ci --features=ci -- --test-threads=1
-          INCLUDE_TPCH=true cargo test --features backtrace,parquet_encryption --profile ci --package datafusion-sqllogictest --test sqllogictests
+          INCLUDE_TPCH=true cargo test --features backtrace,parquet_encryption,substrait --profile ci --package datafusion-sqllogictest --test sqllogictests
       - name: Verify Working Directory Clean
         run: git diff --exit-code
 
@@ -537,7 +537,7 @@ jobs:
         #  command cannot be run for all the .slt files. Run it for just one that works (limit.slt)
         #  until most of the tickets in https://github.com/apache/datafusion/issues/16248 are addressed
         #  and this command can be run without filters.
-        run: cargo test --test sqllogictests -- --substrait-round-trip limit.slt
+        run: cargo test -p datafusion-sqllogictest --test sqllogictests --features substrait -- --substrait-round-trip limit.slt
 
   #  Temporarily commenting out the Windows flow, the reason is enormously slow running build
   #  Waiting for new Windows 2025 github runner
@@ -570,7 +570,7 @@ jobs:
         uses: ./.github/actions/setup-macos-aarch64-builder
       - name: Run tests (excluding doctests)
         shell: bash
-        run: cargo test --profile ci --exclude datafusion-cli --workspace --lib --tests --bins --features avro,json,backtrace,integration-tests
+        run: cargo test --profile ci --exclude datafusion-cli --workspace --lib --tests --bins --features avro,json,backtrace,integration-tests,substrait
 
   vendor:
     name: Verify Vendored Code

datafusion/sqllogictest/Cargo.toml

@@ -48,7 +48,7 @@ chrono = { workspace = true, optional = true }
 clap = { version = "4.5.60", features = ["derive", "env"] }
 datafusion = { workspace = true, default-features = true, features = ["avro"] }
 datafusion-spark = { workspace = true, features = ["core"] }
-datafusion-substrait = { workspace = true, default-features = true }
+datafusion-substrait = { workspace = true, default-features = true, optional = true }
 futures = { workspace = true }
 half = { workspace = true, default-features = true }
 indicatif = "0.18"
@@ -79,6 +79,7 @@ postgres = [
 parquet_encryption = [
     "datafusion/parquet_encryption",
 ]
+substrait = ["datafusion-substrait"]
 
 [dev-dependencies]
 env_logger = { workspace = true }

datafusion/sqllogictest/bin/sqllogictests.rs

@@ -19,11 +19,13 @@ use clap::{ColorChoice, Parser};
 use datafusion::common::instant::Instant;
 use datafusion::common::utils::get_available_parallelism;
 use datafusion::common::{DataFusionError, Result, exec_datafusion_err, exec_err};
+#[cfg(feature = "substrait")]
+use datafusion_sqllogictest::DataFusionSubstraitRoundTrip;
 use datafusion_sqllogictest::TestFile;
 use datafusion_sqllogictest::{
-    CurrentlyExecutingSqlTracker, DataFusion, DataFusionSubstraitRoundTrip, Filter,
-    TestContext, df_value_validator, read_dir_recursive, setup_scratch_dir,
-    should_skip_file, should_skip_record, value_normalizer,
+    CurrentlyExecutingSqlTracker, DataFusion, Filter, TestContext, df_value_validator,
+    read_dir_recursive, setup_scratch_dir, should_skip_file, should_skip_record,
+    value_normalizer,
 };
 use futures::stream::StreamExt;
 use indicatif::{
@@ -426,6 +428,7 @@ fn is_env_truthy(name: &str) -> bool {
         })
 }
 
+#[cfg(feature = "substrait")]
 async fn run_test_file_substrait_round_trip(
     test_file: TestFile,
     validator: Validator,
@@ -468,6 +471,19 @@ async fn run_test_file_substrait_round_trip(
     res
 }
 
+#[cfg(not(feature = "substrait"))]
+async fn run_test_file_substrait_round_trip(
+    _test_file: TestFile,
+    _validator: Validator,
+    _mp: MultiProgress,
+    _mp_style: ProgressStyle,
+    _filters: &[Filter],
+    _currently_executing_sql_tracker: CurrentlyExecutingSqlTracker,
+    _colored_output: bool,
+) -> Result<()> {
+    exec_err!("Cannot run substrait round-trip: the 'substrait' feature is not enabled")
+}
+
 async fn run_test_file(
     test_file: TestFile,
     validator: Validator,

datafusion/sqllogictest/src/engines/mod.rs

@@ -19,13 +19,15 @@
 mod conversion;
 mod currently_executed_sql;
 mod datafusion_engine;
+#[cfg(feature = "substrait")]
 mod datafusion_substrait_roundtrip_engine;
 mod output;
 
 pub use datafusion_engine::DFSqlLogicTestError;
 pub use datafusion_engine::DataFusion;
 pub use datafusion_engine::convert_batches;
 pub use datafusion_engine::convert_schema_to_types;
+#[cfg(feature = "substrait")]
 pub use datafusion_substrait_roundtrip_engine::DataFusionSubstraitRoundTrip;
 pub use output::DFColumnType;
 pub use output::DFOutput;

datafusion/sqllogictest/src/lib.rs

@@ -34,6 +34,7 @@ pub use engines::DFColumnType;
 pub use engines::DFOutput;
 pub use engines::DFSqlLogicTestError;
 pub use engines::DataFusion;
+#[cfg(feature = "substrait")]
 pub use engines::DataFusionSubstraitRoundTrip;
 pub use engines::convert_batches;
 pub use engines::convert_schema_to_types;