Skip to content

[HSTACK] Delta-rs controls stats_parsed loading in the kernel #19

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
aditanase wants to merge 21 commits into from
Closed

[HSTACK] Delta-rs controls stats_parsed loading in the kernel #19

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
21 commits
Select commit Hold shift + click to select a range
97279e0
[HSTACK] chore: datafusion 54, buoyant_kernel 0.25.1 main Cargo.toml …
adragomir Jul 22, 2026
4524c07
[HSTACK] chore: expose extra methods: scan, DeltaScan struct properti…
adragomir Jul 7, 2026
ea3359c
[HSTACK] fix: optimize scan builder limit pushdown
aditanase Jul 7, 2026
b63e470
[HSTACK] fix: REEVALUATE, bring back old table provider
adragomir Jul 7, 2026
668a053
[HSTACK] feat: add delta table udtf and register function
adragomir Feb 6, 2026
162a269
[HSTACK] test: add support for deep projections TEST
adragomir Feb 6, 2026
f2624f8
[HSTACK] fix: schema mapping for record batches with an empty schema …
adragomir Jul 7, 2026
6611ae1
[HSTACK] feat: log segment size limiter
ccciudatu May 7, 2026
96edf4f
[HSTACK] feat: expose log metadata and total size to support weighted…
aditanase Feb 6, 2026
841c803
[HSTACK] feat: add support for log_store reconfigure
aditanase Feb 6, 2026
ebd223f
[HSTACK] fix: add table constructor that does not remove final `/`
adragomir Jul 7, 2026
f530379
[HSTACK] feat: implement physical codec for "next" version of Executi…
adragomir Aug 27, 2026
2982a1f
[HSTACK] feat: add option to modify the low-level Delta schema to mak…
adragomir Aug 13, 2026
d7ac2fb
[HSTACK] feat: expose snapshot load metrics in order to detect if log…
adragomir Aug 27, 2026
556b0fa
[HSTACK] feat: delta engine injection support
Jul 7, 2026
9603b8e
[HSTACK] feat: add ability to construct Snapshot and EagerSnapshot ma…
adragomir Jul 7, 2026
9b62d6a
[HSTACK] chore: downgrade time dep to overcome incompat with lancedb
adragomir Jul 23, 2026
109e6be
[HSTACK] Delta-rs controls stats_parsed loading in the kernel
adragomir Aug 13, 2026
868f154
[HSTACK] feat: make to_datafusion_expr public
adragomir Aug 20, 2026
083bce8
[HSTACK][TEST] feat: Add predicate test
adragomir Aug 20, 2026
984edfe
[HSTACK] Disable schema_force_view_types completely
adragomir Sep 23, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
85 changes: 80 additions & 5 deletions Cargo.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,14 +16,14 @@ documentation = "https://docs.rs/deltalake"
repository = "https://github.com/delta-io/delta.rs"

[workspace.dependencies]
#delta_kernel = { package = "buoyant_kernel", path = "../delta-kernel-rs/kernel", features = [
delta_kernel = { package = "buoyant_kernel", git = "https://github.com/buoyant-data/delta-kernel-rs", branch = "buoyant/main", features = [
# delta_kernel = { package = "buoyant_kernel", path = "../delta-kernel-rs/kernel", features = [
delta_kernel = { package = "buoyant_kernel", git = "https://github.com/hstack/delta-kernel-rs", branch = "buoyant-v0.25.1-stats", features = [
#delta_kernel = { package = "buoyant_kernel", version = "0.25.0,<0.25.100", features = [
"arrow-58",
"internal-api",
] }
#delta_kernel_default_engine = { package = "buoyant_kernel_engine", path = "../delta-kernel-rs/default-engine", features = ["arrow-58", "rustls"], default-features = false }
delta_kernel_default_engine = { package = "buoyant_kernel_engine", git = "https://github.com/buoyant-data/delta-kernel-rs", branch = "buoyant/main", features = ["arrow-58", "rustls"], default-features = false }
# delta_kernel_default_engine = { package = "buoyant_kernel_engine", path = "../delta-kernel-rs/default-engine", features = ["arrow-58", "rustls"], default-features = false }
delta_kernel_default_engine = { package = "buoyant_kernel_engine", git = "https://github.com/hstack/delta-kernel-rs", branch = "buoyant-v0.25.1-stats", features = ["arrow-58", "rustls"], default-features = false }
#delta_kernel_default_engine = { package = "buoyant_kernel_engine", version = "0.25.0,<0.25.100", features = ["arrow-58", "rustls"], default-features = false }


Expand Down Expand Up @@ -52,11 +52,13 @@ datafusion-proto = { version = "54.0.0" }
# serde
serde = { version = "1.0.194", features = ["derive"] }
serde_json = "1"
strum = { version = "0.28", features = ["derive"] }
prost = { version = "0.14.3" }

# "stdlib"
bytes = { version = "1" }
chrono = { version = "0.4.40", default-features = false, features = ["clock"] }
time = { version = "=0.3.53" }
time = { version = "=0.3.47" }
tracing = { version = "0.1", features = ["log"] }
regex = { version = "1" }
thiserror = { version = "2" }
Expand Down Expand Up @@ -160,3 +162,76 @@ inherits = "release"
opt-level = 3
codegen-units = 1
lto = "thin"

[patch.crates-io]
datafusion = { git = 'https://github.com/hstack/datafusion.git', branch = 'main-54' }
datafusion-catalog = { git = 'https://github.com/hstack/datafusion.git', branch = 'main-54' }
datafusion-catalog-listing = { git = 'https://github.com/hstack/datafusion.git', branch = 'main-54' }
datafusion-common = { git = 'https://github.com/hstack/datafusion.git', branch = 'main-54' }
datafusion-common-runtime = { git = 'https://github.com/hstack/datafusion.git', branch = 'main-54' }
datafusion-datasource = { git = 'https://github.com/hstack/datafusion.git', branch = 'main-54' }
datafusion-datasource-avro = { git = 'https://github.com/hstack/datafusion.git', branch = 'main-54' }
datafusion-datasource-csv = { git = 'https://github.com/hstack/datafusion.git', branch = 'main-54' }
datafusion-datasource-json = { git = 'https://github.com/hstack/datafusion.git', branch = 'main-54' }
datafusion-datasource-parquet = { git = 'https://github.com/hstack/datafusion.git', branch = 'main-54' }
datafusion-execution = { git = 'https://github.com/hstack/datafusion.git', branch = 'main-54' }
datafusion-expr = { git = 'https://github.com/hstack/datafusion.git', branch = 'main-54' }
datafusion-expr-common = { git = 'https://github.com/hstack/datafusion.git', branch = 'main-54' }
datafusion-ffi = { git = 'https://github.com/hstack/datafusion.git', branch = 'main-54' }
datafusion-functions = { git = 'https://github.com/hstack/datafusion.git', branch = 'main-54' }
datafusion-functions-aggregate = { git = 'https://github.com/hstack/datafusion.git', branch = 'main-54' }
datafusion-functions-aggregate-common = { git = 'https://github.com/hstack/datafusion.git', branch = 'main-54' }
datafusion-functions-nested = { git = 'https://github.com/hstack/datafusion.git', branch = 'main-54' }
datafusion-functions-table = { git = 'https://github.com/hstack/datafusion.git', branch = 'main-54' }
datafusion-functions-window = { git = 'https://github.com/hstack/datafusion.git', branch = 'main-54' }
datafusion-functions-window-common = { git = 'https://github.com/hstack/datafusion.git', branch = 'main-54' }
datafusion-optimizer = { git = 'https://github.com/hstack/datafusion.git', branch = 'main-54' }
datafusion-physical-expr = { git = 'https://github.com/hstack/datafusion.git', branch = 'main-54' }
datafusion-physical-expr-adapter = { git = 'https://github.com/hstack/datafusion.git', branch = 'main-54' }
datafusion-physical-expr-common = { git = 'https://github.com/hstack/datafusion.git', branch = 'main-54' }
datafusion-physical-optimizer = { git = 'https://github.com/hstack/datafusion.git', branch = 'main-54' }
datafusion-pruning = { git = 'https://github.com/hstack/datafusion.git', branch = 'main-54' }
datafusion-physical-plan = { git = 'https://github.com/hstack/datafusion.git', branch = 'main-54' }
datafusion-proto = { git = 'https://github.com/hstack/datafusion.git', branch = 'main-54' }
datafusion-proto-common = { git = 'https://github.com/hstack/datafusion.git', branch = 'main-54' }
datafusion-session = { git = 'https://github.com/hstack/datafusion.git', branch = 'main-54' }
datafusion-spark = { git = 'https://github.com/hstack/datafusion.git', branch = 'main-54' }
datafusion-sql = { git = 'https://github.com/hstack/datafusion.git', branch = 'main-54' }
datafusion-substrait = { git = 'https://github.com/hstack/datafusion.git', branch = 'main-54' }
#datafusion = { path = "../datafusion/datafusion/core" }
#datafusion-catalog = { path = "../datafusion/datafusion/catalog" }
#datafusion-catalog-listing = { path = "../datafusion/datafusion/catalog-listing" }
#datafusion-common = { path = "../datafusion/datafusion/common" }
#datafusion-common-runtime = { path = "../datafusion/datafusion/common-runtime" }
#datafusion-datasource = { path = "../datafusion/datafusion/datasource" }
#datafusion-datasource-arrow = { path = "../datafusion/datafusion/datasource-arrow" }
#datafusion-datasource-avro = { path = "../datafusion/datafusion/datasource-avro" }
#datafusion-datasource-csv = { path = "../datafusion/datafusion/datasource-csv" }
#datafusion-datasource-json = { path = "../datafusion/datafusion/datasource-json" }
#datafusion-datasource-parquet = { path = "../datafusion/datafusion/datasource-parquet" }
#datafusion-execution = { path = "../datafusion/datafusion/execution" }
#datafusion-expr = { path = "../datafusion/datafusion/expr" }
#datafusion-expr-common = { path = "../datafusion/datafusion/expr-common" }
#datafusion-ffi = { path = "../datafusion/datafusion/ffi" }
#datafusion-functions = { path = "../datafusion/datafusion/functions" }
#datafusion-functions-aggregate = { path = "../datafusion/datafusion/functions-aggregate" }
#datafusion-functions-aggregate-common = { path = "../datafusion/datafusion/functions-aggregate-common" }
#datafusion-functions-nested = { path = "../datafusion/datafusion/functions-nested" }
#datafusion-functions-table = { path = "../datafusion/datafusion/functions-table" }
#datafusion-functions-window = { path = "../datafusion/datafusion/functions-window" }
#datafusion-functions-window-common = { path = "../datafusion/datafusion/functions-window-common" }
#datafusion-optimizer = { path = "../datafusion/datafusion/optimizer" }
#datafusion-physical-expr = { path = "../datafusion/datafusion/physical-expr" }
#datafusion-physical-expr-adapter = { path = "../datafusion/datafusion/physical-expr-adapter" }
#datafusion-physical-expr-common = { path = "../datafusion/datafusion/physical-expr-common" }
#datafusion-physical-optimizer = { path = "../datafusion/datafusion/physical-optimizer" }
#datafusion-physical-plan = { path = "../datafusion/datafusion/physical-plan" }
#datafusion-proto = { path = "../datafusion/datafusion/proto" }
#datafusion-proto-common = { path = "../datafusion/datafusion/proto-common" }
#datafusion-pruning = { path = "../datafusion/datafusion/pruning" }
#datafusion-session = { path = "../datafusion/datafusion/session" }
#datafusion-spark = { path = "../datafusion/datafusion/spark" }
#datafusion-sql = { path = "../datafusion/datafusion/sql" }

#buoyant_kernel = { git = 'https://github.com/hstack/delta-kernel-rs.git', branch = 'buoyant-v0.25.1-stats' }
#buoyant_kernel = { path = "../delta-kernel-rs/kernel" }
3 changes: 3 additions & 0 deletions crates/core/Cargo.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,8 @@ datafusion-proto = { workspace = true, optional = true }
# serde
serde = { workspace = true, features = ["derive"] }
serde_json = { workspace = true }
strum = { workspace = true, features = ["derive"] }
prost = { workspace = true }

# "stdlib"
bytes = { workspace = true }
Expand Down Expand Up @@ -95,6 +97,7 @@ rand = "0.10"
sqlparser = { version = "0.61.0" }
humantime = { version = "2.1.0", optional = true }
validator = { version = "0.19", features = ["derive"] }
ctor = "0.10"

[dev-dependencies]
arrow = { workspace = true, features = ["prettyprint"]}
Expand Down
2 changes: 1 addition & 1 deletion crates/core/src/delta_datafusion/find_files.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -215,7 +215,7 @@ impl TreeNodeVisitor<'_> for FindFilesExprProperties {
/// rather than materializing every partition column — which avoids coercing values the
/// predicate never reads and prevents an unrelated/malformed partition value from
/// weakening pruning.
pub(crate) fn extract_partition_only_predicate(
pub fn extract_partition_only_predicate(
predicate: Expr,
partition_columns: &[String],
) -> DeltaResult<Option<(Expr, Vec<String>)>> {
Expand Down
22 changes: 20 additions & 2 deletions crates/core/src/delta_datafusion/mod.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -73,15 +73,25 @@ pub(crate) use column_mapping::ColumnMappingState;
pub(crate) use data_validation::{
DataValidationExec, constraints_to_exprs, generated_columns_to_exprs, validation_predicates,
};
pub use engine::DataFusionEngine;
pub(crate) use find_files::*;
pub(crate) use table_provider::next::normalize_path_as_file_id;
pub use table_provider::{
DeltaScanConfig, DeltaScanConfigBuilder, TableProviderBuilder, next::DeltaScanExec,
next::DeltaNextPhysicalCodec,
};
pub(crate) use table_provider::{
next::FILE_ID_COLUMN_DEFAULT, resolve_file_column_name, update_datafusion_session,
};

/// Converts a Delta Kernel expression to a DataFusion expression.
pub fn to_datafusion_expr(
expr: &delta_kernel::expressions::Expression,
output_type: &delta_kernel::schema::DataType,
) -> DataFusionResult<Expr> {
engine::to_datafusion_expr(expr, output_type)
}

pub(crate) const PATH_COLUMN: &str = "__delta_rs_path";

#[doc(hidden)]
Expand All @@ -100,8 +110,13 @@ pub mod planner;
mod session;
pub use session::SessionFallbackPolicy;
pub(crate) use session::{SessionResolveContext, resolve_session_state};
use crate::delta_datafusion::schema_null::rewrite_schema_with_nullable_fields;

mod table_provider;
pub mod udtf;
pub(crate) mod utils;
pub mod table_provider_old;
pub mod schema_null;

impl From<DeltaTableError> for DataFusionError {
fn from(err: DeltaTableError) -> Self {
Expand Down Expand Up @@ -232,7 +247,7 @@ fn _arrow_schema(
partition_columns: &[String],
wrap_partitions: bool,
) -> ArrowSchemaRef {
let fields = schema
let mut fields = schema
.fields()
.into_iter()
.filter(|f| !partition_columns.contains(&f.name().to_string()))
Expand Down Expand Up @@ -261,7 +276,10 @@ fn _arrow_schema(
}),
)
.collect::<Vec<_>>();
Arc::new(ArrowSchema::new(fields))
let mut schema = Arc::new(ArrowSchema::new(fields));
// @Hstack - add the option to have an env var that can nullify fields in the delta schema
schema = rewrite_schema_with_nullable_fields(schema);
schema
}

pub(crate) fn files_matching_predicate<'a>(
Expand Down
Loading
Loading