diff --git a/vortex-datafusion/src/persistent/opener.rs b/vortex-datafusion/src/persistent/opener.rs index 5986a06da49..95ae9f789e7 100644 --- a/vortex-datafusion/src/persistent/opener.rs +++ b/vortex-datafusion/src/persistent/opener.rs @@ -352,9 +352,7 @@ impl FileOpener for VortexOpener { scan_builder = scan_builder.with_limit(limit); } - if let Some(concurrency) = scan_concurrency { - scan_builder = scan_builder.with_concurrency(concurrency); - } + scan_builder = scan_builder.with_concurrency(scan_concurrency.unwrap_or(4)); let stream = scan_builder .with_metrics_registry(metrics_registry) diff --git a/vortex-scan/src/scan_builder.rs b/vortex-scan/src/scan_builder.rs index 4c4548cc665..95ef40aea0a 100644 --- a/vortex-scan/src/scan_builder.rs +++ b/vortex-scan/src/scan_builder.rs @@ -349,10 +349,10 @@ impl Stream for LazyScanStream { LazyScanState::Builder(builder) => { let builder = builder.take().vortex_expect("polled after completion"); let ordered = builder.ordered; - let num_workers = std::thread::available_parallelism() - .map(|n| n.get()) - .unwrap_or(1); - let concurrency = builder.concurrency * num_workers; + // let num_workers = std::thread::available_parallelism() + // .map(|n| n.get()) + // .unwrap_or(1); + let concurrency = builder.concurrency; let handle = builder.session.handle(); let task = handle.spawn_blocking(move || { builder.prepare().and_then(|scan| scan.execute(None))