chore: bump kernel, datafusion 44rc1

Signed-off-by: Ion Koutsouris <[email protected]>

Cargo.toml

@@ -26,7 +26,7 @@ debug = true
 debug = "line-tables-only"
 
 [workspace.dependencies]
-delta_kernel = { version = "0.5.0", features = ["default-engine"] }
+delta_kernel = { version = "0.6.0", features = ["default-engine"] }
 #delta_kernel = { path = "../delta-kernel-rs/kernel", features = ["sync-engine"]  }
 
 # arrow
@@ -45,16 +45,16 @@ object_store = { version = "0.11.2" }
 parquet = { version = "53" }
 
 # datafusion
-datafusion = { version = "43" }
-datafusion-expr = { version = "43" }
-datafusion-common = { version = "43" }
-datafusion-ffi = { version = "43" }
-datafusion-functions = { version = "43" }
-datafusion-functions-aggregate = { version = "43" }
-datafusion-physical-expr = { version = "43" }
-datafusion-physical-plan = { version = "43" }
-datafusion-proto = { version = "43" }
-datafusion-sql = { version = "43" }
+datafusion = { version = "44" }
+datafusion-expr = { version = "44" }
+datafusion-common = { version = "44" }
+datafusion-ffi = { version = "44" }
+datafusion-functions = { version = "44" }
+datafusion-functions-aggregate = { version = "44" }
+datafusion-physical-expr = { version = "44" }
+datafusion-physical-plan = { version = "44" }
+datafusion-proto = { version = "44" }
+datafusion-sql = { version = "44" }
 
 # serde
 serde = { version = "1.0.194", features = ["derive"] }
@@ -69,24 +69,10 @@ thiserror = { version = "2" }
 url = { version = "2" }
 urlencoding = "2.1.3"
 uuid = { version = "1" }
-path-tree = { version = "0.8.1"} # pin to 0.8.1 due to nightly features
 
 # runtime / async
 async-trait = { version = "0.1" }
 futures = { version = "0.3" }
 tokio = { version = "1" }
 num_cpus = { version = "1" }
 
-# temporary datafusion patches
-[patch.crates-io]
-datafusion = { git = "https://github.com/apache/datafusion.git", rev = "a50ed3488f77743a192d9e8dd9c99f00df659ef1" }
-datafusion-common = { git = "https://github.com/apache/datafusion.git", rev = "a50ed3488f77743a192d9e8dd9c99f00df659ef1" }
-datafusion-execution = { git = "https://github.com/apache/datafusion.git", rev = "a50ed3488f77743a192d9e8dd9c99f00df659ef1" }
-datafusion-expr = { git = "https://github.com/apache/datafusion.git", rev = "a50ed3488f77743a192d9e8dd9c99f00df659ef1" }
-datafusion-ffi = { git = "https://github.com/apache/datafusion.git", rev = "a50ed3488f77743a192d9e8dd9c99f00df659ef1" }
-datafusion-functions = { git = "https://github.com/apache/datafusion.git", rev = "a50ed3488f77743a192d9e8dd9c99f00df659ef1" }
-datafusion-functions-aggregate = { git = "https://github.com/apache/datafusion.git", rev = "a50ed3488f77743a192d9e8dd9c99f00df659ef1" }
-datafusion-physical-expr = { git = "https://github.com/apache/datafusion.git", rev = "a50ed3488f77743a192d9e8dd9c99f00df659ef1" }
-datafusion-physical-plan = { git = "https://github.com/apache/datafusion.git", rev = "a50ed3488f77743a192d9e8dd9c99f00df659ef1" }
-datafusion-proto = { git = "https://github.com/apache/datafusion.git", rev = "a50ed3488f77743a192d9e8dd9c99f00df659ef1" }
-datafusion-sql = { git = "https://github.com/apache/datafusion.git", rev = "a50ed3488f77743a192d9e8dd9c99f00df659ef1" }

crates/aws/src/lib.rs

@@ -78,7 +78,7 @@ impl LogStoreFactory for S3LogStoreFactory {
                 store,
             )?));
         }
-        Ok(default_logstore(store, location, &options))
+        Ok(default_logstore(store, location, options))
     }
 }
 
@@ -141,6 +141,7 @@ impl std::fmt::Debug for DynamoDbLockClient {
 
 impl DynamoDbLockClient {
     /// Creates a new DynamoDbLockClient from the supplied storage options.
+    #[allow(clippy::too_many_arguments)]
     pub fn try_new(
         sdk_config: &SdkConfig,
         lock_table_name: Option<String>,

crates/azure/Cargo.toml

@@ -14,7 +14,7 @@ rust-version.workspace = true
 [dependencies]
 deltalake-core = { version = "0.23.0", path = "../core", features = [
     "datafusion",
-] }
+]}
 lazy_static = "1"
 
 # workspace depenndecies

crates/catalog-unity/Cargo.toml

@@ -17,17 +17,19 @@ tokio.workspace = true
 serde.workspace = true
 serde_json.workspace = true
 thiserror.workspace = true
-deltalake-core = { version = "0.23", path = "../core" }
+deltalake-core = { version = "0.23", path = "../core", features = [
+    "datafusion",
+]}
 reqwest = { version = "0.12", default-features = false, features = ["rustls-tls", "json", "http2"] }
 reqwest-retry = "0.7"
 reqwest-middleware = "0.4.0"
 rand = "0.8"
-futures = "0.3"
-chrono = "0.4"
+futures = { workspace = true }
+chrono = { workspace = true }
 dashmap = "6"
-tracing = "0.1"
-datafusion = { version = "43", optional = true }
-datafusion-common = { version = "43", optional = true }
+tracing = { workspace = true }
+datafusion = { workspace = true, optional = true }
+datafusion-common = { workspace = true, optional = true }
 
 [dev-dependencies]
 tokio = { version = "1", features = ["macros", "rt-multi-thread"] }

crates/catalog-unity/src/credential.rs

@@ -205,8 +205,7 @@ impl TokenCredential for AzureCliCredential {
                             "got unexpected token type from azure cli: {0}",
                             token_response.token_type
                         ),
-                    }
-                    .into());
+                    });
                 }
                 let duration =
                     token_response.expires_on.naive_local() - chrono::Local::now().naive_local();
@@ -224,18 +223,15 @@ impl TokenCredential for AzureCliCredential {
                 let message = String::from_utf8_lossy(&az_output.stderr);
                 Err(UnityCatalogError::AzureCli {
                     message: message.into(),
-                }
-                .into())
+                })
             }
             Err(e) => match e.kind() {
                 std::io::ErrorKind::NotFound => Err(UnityCatalogError::AzureCli {
                     message: "Azure Cli not installed".into(),
-                }
-                .into()),
+                }),
                 error_kind => Err(UnityCatalogError::AzureCli {
                     message: format!("io error: {error_kind:?}"),
-                }
-                .into()),
+                }),
             },
         }
     }

crates/catalog-unity/src/lib.rs

@@ -644,8 +644,7 @@ impl DataCatalog for UnityCatalog {
             GetTableResponse::Error(err) => Err(UnityCatalogError::InvalidTable {
                 error_code: err.error_code,
                 message: err.message,
-            }
-            .into()),
+            }),
         }
     }
 }

crates/core/src/table/state_arrow.rs

@@ -14,7 +14,7 @@ use arrow_array::{
 use arrow_cast::cast;
 use arrow_cast::parse::Parser;
 use arrow_schema::{DataType, Field, Fields, TimeUnit};
-use delta_kernel::table_features::ColumnMappingMode;
+use delta_kernel::table_features::{validate_schema_column_mapping, ColumnMappingMode};
 use itertools::Itertools;
 
 use super::state::DeltaTableState;
@@ -171,6 +171,8 @@ impl DeltaTableState {
             })
             .collect::<HashMap<&str, _>>();
 
+        validate_schema_column_mapping(self.schema(), column_mapping_mode)?;
+
         let physical_name_to_logical_name = match column_mapping_mode {
             ColumnMappingMode::None => HashMap::with_capacity(0), // No column mapping, no need for this HashMap
             ColumnMappingMode::Id | ColumnMappingMode::Name => metadata
@@ -184,7 +186,7 @@ impl DeltaTableState {
                             "Invalid partition column {0}",
                             name
                         )))?
-                        .physical_name(column_mapping_mode)?
+                        .physical_name()
                         .to_string();
                     Ok((physical_name, name.as_str()))
                 })

python/src/lib.rs

@@ -670,6 +670,7 @@ impl RawDeltaTable {
     }
 
     #[pyo3(signature = (starting_version = 0, ending_version = None, starting_timestamp = None, ending_timestamp = None, columns = None, allow_out_of_range = false))]
+    #[allow(clippy::too_many_arguments)]
     pub fn load_cdf(
         &mut self,
         py: Python,