diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3b75498a9..9d9e69d0b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -227,31 +227,21 @@ jobs: include: - suite: rust needs_docker: true - needs_lumina: false needs_uv: false - suite: datafusion needs_docker: true - needs_lumina: true - needs_uv: false - - suite: lumina - needs_docker: false - needs_lumina: true needs_uv: false - suite: python needs_docker: true - needs_lumina: false needs_uv: true - suite: go needs_docker: true - needs_lumina: false needs_uv: false - suite: jindo needs_docker: false - needs_lumina: false needs_uv: false - suite: oss-cpp needs_docker: false - needs_lumina: false needs_uv: false env: RUST_LOG: DEBUG @@ -278,30 +268,6 @@ jobs: if: matrix.suite == 'rust' run: cargo test --locked -p paimon-integration-tests --all-targets - - name: Install lumina native library - if: matrix.needs_lumina == true - run: | - pip install lumina-data - echo "LUMINA_LIB_PATH=$(python3 -c 'import lumina_data; print(lumina_data.__path__[0])')/lib/liblumina_py.so" >> $GITHUB_ENV - - - name: Core Lumina Native Build Test - if: matrix.suite == 'lumina' - run: > - cargo test --locked -p paimon --lib - table::lumina_index_build_builder::tests::test_execute_writes_lumina_index_manifest - --features fulltext,vortex - -- --ignored --exact - - # The test filter selects tests, not targets, so without `--test` cargo links all 26 - # integration binaries with `vortex` enabled just to run one ignored test in one of them. - - name: DataFusion Lumina Build Query E2E Test - if: matrix.suite == 'lumina' - run: > - cargo test --locked -p paimon-datafusion --test read_tables - --features vortex - vector_search_tests::test_lumina_build_then_vector_search_query - -- --ignored --exact - - name: DataFusion Integration Test if: matrix.suite == 'datafusion' env: diff --git a/crates/integrations/datafusion/src/procedures.rs b/crates/integrations/datafusion/src/procedures.rs index 99439c35d..160fd34b1 100644 --- a/crates/integrations/datafusion/src/procedures.rs +++ b/crates/integrations/datafusion/src/procedures.rs @@ -31,8 +31,7 @@ //! - `CALL sys.create_global_index(table => '...', index_column => '...', index_type => 'btree')` //! - `CALL sys.create_global_index(table => '...', index_column => '...', index_type => 'bitmap')` //! - `CALL sys.create_global_index(table => '...', index_column => '...', index_type => 'ivf-pq')` -//! - `CALL sys.drop_global_index(table => '...', index_column => '...', index_type => 'btree')` (also 'bitmap', 'multivalue', 'fm', 'lumina', or a vindex type such as 'ivf-pq') -//! - `CALL sys.create_lumina_index(table => '...', index_column => '...')` +//! - `CALL sys.drop_global_index(table => '...', index_column => '...', index_type => 'btree')` (also 'bitmap', 'multivalue', 'fm', or a vindex type such as 'diskann') //! //! REST management procedures (REST catalogs only, mirroring Java's //! `RESTCatalog.permissionManagement()` / `policyManagement()`): @@ -74,7 +73,6 @@ use paimon::api::{ PermissionColumns, PermissionResource, PolicyType, ResourceType, RowFilter, }; use paimon::catalog::{Catalog, Identifier, RESTCatalog}; -use paimon::lumina::{is_lumina_index_type, LUMINA_IDENTIFIER}; use paimon::spec::{CoreOptions, DataType, Datum, Predicate, PredicateBuilder, Snapshot}; use paimon::table::{ normalize_global_index_type, BranchManager, ConsumerManager, SnapshotManager, Table, @@ -224,7 +222,6 @@ fn declared_parameters(proc_name: &str) -> Option<&'static [&'static str]> { "partitions", "dry_run", ], - "create_lumina_index" => &["table", "index_column", "index_type", "options"], "grant_permission" => &[ "resource_type", "access", @@ -347,7 +344,6 @@ pub async fn execute_procedure( "delete_branch" => proc_delete_branch(ctx, catalog, catalog_name, &args).await, "create_global_index" => proc_create_global_index(ctx, catalog, catalog_name, &args).await, "drop_global_index" => proc_drop_global_index(ctx, catalog, catalog_name, &args).await, - "create_lumina_index" => proc_create_lumina_index(ctx, catalog, catalog_name, &args).await, "grant_permission" => proc_grant_permission(ctx, catalog, catalog_name, &args).await, "revoke_permission" => proc_revoke_permission(ctx, catalog, catalog_name, &args).await, "list_permissions" => proc_list_permissions(ctx, catalog, catalog_name, &args).await, @@ -1018,34 +1014,6 @@ async fn proc_create_tag_from_watermark( ok_result(ctx) } -async fn proc_create_lumina_index( - ctx: &SessionContext, - catalog: &Arc, - catalog_name: &str, - args: &HashMap, -) -> DFResult { - let table = get_table(catalog, catalog_name, args).await?; - let index_column = require_arg(args, "index_column")?; - let mut builder = table.new_global_index_build_builder(); - builder.with_index_column(index_column); - let index_type = normalize_index_type( - args.get("index_type") - .map(String::as_str) - .unwrap_or(LUMINA_IDENTIFIER), - ); - if !is_lumina_index_type(&index_type) { - return Err(DataFusionError::NotImplemented(format!( - "Unsupported Lumina index type: {index_type}" - ))); - } - builder.with_index_type(&index_type); - if let Some(options) = args.get("options") { - builder.with_options(parse_key_value_options(options)?); - } - builder.execute().await.map_err(to_datafusion_error)?; - ok_result(ctx) -} - async fn proc_create_global_index( ctx: &SessionContext, catalog: &Arc, @@ -2104,10 +2072,7 @@ mod tests { assert_eq!(normalize_index_type(" BitMap\t"), "bitmap"); assert_eq!(normalize_index_type("IVF-FLAT"), "ivf-flat"); assert_eq!(normalize_index_type("Ivf-Pq"), "ivf-pq"); - assert_eq!( - normalize_index_type("Lumina-Vector-Ann"), - "lumina-vector-ann" - ); + assert_eq!(normalize_index_type("DiskANN"), "diskann"); // Already canonical values are returned unchanged, and an unknown type // is normalized but not rewritten -- the caller still rejects it. assert_eq!(normalize_index_type("ivf-flat"), "ivf-flat"); diff --git a/crates/integrations/datafusion/testdata/test_lumina_vector.tar.gz b/crates/integrations/datafusion/testdata/test_lumina_vector.tar.gz deleted file mode 100644 index 3909f95cf..000000000 Binary files a/crates/integrations/datafusion/testdata/test_lumina_vector.tar.gz and /dev/null differ diff --git a/crates/integrations/datafusion/tests/procedures.rs b/crates/integrations/datafusion/tests/procedures.rs index 5279155a0..8e5cc26a2 100644 --- a/crates/integrations/datafusion/tests/procedures.rs +++ b/crates/integrations/datafusion/tests/procedures.rs @@ -716,42 +716,6 @@ async fn test_delete_branch_rejects_path_separated_name() { table.copy_with_branch("prod").await.unwrap(); } -#[tokio::test] -async fn test_create_lumina_index_requires_index_column() { - let (_tmp, sql_context) = setup_table_with_snapshots().await; - - assert_sql_error( - &sql_context, - "CALL sys.create_lumina_index(table => 'test_db.t1')", - "Missing required argument: 'index_column'", - ) - .await; -} - -#[tokio::test] -async fn test_create_lumina_index_rejects_invalid_index_type() { - let (_tmp, sql_context) = setup_table_with_snapshots().await; - - assert_sql_error( - &sql_context, - "CALL sys.create_lumina_index(table => 'test_db.t1', index_column => 'name', index_type => 'btree')", - "Unsupported Lumina index type: btree", - ) - .await; -} - -#[tokio::test] -async fn test_create_lumina_index_rejects_invalid_options() { - let (_tmp, sql_context) = setup_table_with_snapshots().await; - - assert_sql_error( - &sql_context, - "CALL sys.create_lumina_index(table => 'test_db.t1', index_column => 'name', options => 'lumina.index.dimension')", - "Expected comma-separated key=value pairs", - ) - .await; -} - async fn setup_btree_global_index_table( table_name: &str, ) -> (tempfile::TempDir, paimon_datafusion::SQLContext) { @@ -838,7 +802,13 @@ async fn test_create_global_index_requires_index_column() { async fn test_create_global_index_rejects_unsupported_index_types() { let (_tmp, sql_context) = setup_btree_global_index_table("global_index_bad_type").await; - for index_type in ["hash", "ivf-hnsw-flat", "ivf-hnsw-sq"] { + for index_type in [ + "hash", + "ivf-hnsw-flat", + "ivf-hnsw-sq", + "lumina", + "lumina-vector-ann", + ] { assert_sql_error( &sql_context, &format!( @@ -854,6 +824,17 @@ async fn test_create_global_index_rejects_unsupported_index_types() { } } +#[tokio::test] +async fn test_removed_vector_procedure_is_unknown() { + let (_tmp, sql_context) = setup_btree_global_index_table("removed_vector_proc").await; + assert_sql_error( + &sql_context, + "CALL sys.create_lumina_index(table => 'test_db.removed_vector_proc', index_column => 'id')", + "Unknown procedure: create_lumina_index", + ) + .await; +} + #[tokio::test] async fn test_create_global_index_rejects_invalid_sorted_options() { let (_tmp, sql_context) = setup_btree_global_index_table("btree_options").await; @@ -2493,19 +2474,19 @@ async fn test_drop_global_index_rejects_unsupported_index_type() { } #[tokio::test] -async fn test_drop_global_index_accepts_lumina_type() { - let (_tmp, sql_context) = setup_btree_global_index_table("lumina_drop_accepted").await; +async fn test_drop_global_index_accepts_diskann_type() { + let (_tmp, sql_context) = setup_btree_global_index_table("diskann_drop_accepted").await; exec( &sql_context, - "INSERT INTO paimon.test_db.lumina_drop_accepted (id, name) VALUES (1, 'alice')", + "INSERT INTO paimon.test_db.diskann_drop_accepted (id, name) VALUES (1, 'alice')", ) .await; - // No lumina index exists; the call must be accepted and succeed (no-match Ok), + // No diskann index exists; the call must be accepted and succeed (no-match Ok), // NOT rejected as an unsupported type. exec( &sql_context, - "CALL sys.drop_global_index(table => 'test_db.lumina_drop_accepted', index_column => 'id', index_type => 'lumina')", + "CALL sys.drop_global_index(table => 'test_db.diskann_drop_accepted', index_column => 'id', index_type => 'diskann')", ) .await; } diff --git a/crates/integrations/datafusion/tests/read_tables.rs b/crates/integrations/datafusion/tests/read_tables.rs index d18311ca3..ef42a2d44 100644 --- a/crates/integrations/datafusion/tests/read_tables.rs +++ b/crates/integrations/datafusion/tests/read_tables.rs @@ -1778,11 +1778,6 @@ mod vector_search_tests { (ctx, catalog, tmp) } - async fn create_lumina_vector_search_context( - ) -> (SQLContext, Arc, tempfile::TempDir) { - create_vector_search_context("test_lumina_vector.tar.gz").await - } - async fn create_java_vindex_vector_search_context( ) -> (SQLContext, Arc, tempfile::TempDir) { create_vector_search_context("test_java_vindex_vector.tar.gz").await @@ -1804,29 +1799,6 @@ mod vector_search_tests { (ctx, catalog, tmp) } - fn build_lumina_table_schema() -> Schema { - let mut options = std::collections::HashMap::new(); - options.insert("row-tracking.enabled".to_string(), "true".to_string()); - options.insert("data-evolution.enabled".to_string(), "true".to_string()); - options.insert("global-index.enabled".to_string(), "true".to_string()); - options.insert( - "global-index.row-count-per-shard".to_string(), - "3".to_string(), - ); - options.insert("lumina.index.dimension".to_string(), "2".to_string()); - options.insert("lumina.encoding.type".to_string(), "rawf32".to_string()); - - Schema::builder() - .column("id", DataType::Int(IntType::new())) - .column( - "embedding", - DataType::Array(ArrayType::new(DataType::Float(FloatType::new()))), - ) - .options(options) - .build() - .expect("Failed to build table schema") - } - fn build_vindex_table_schema() -> Schema { let mut options = std::collections::HashMap::new(); options.insert("row-tracking.enabled".to_string(), "true".to_string()); @@ -1974,9 +1946,9 @@ mod vector_search_tests { #[tokio::test] async fn test_vector_search_top3() { - let (ctx, _catalog, _tmp) = create_lumina_vector_search_context().await; + let (ctx, _catalog, _tmp) = create_java_vindex_vector_search_context().await; let batches = ctx - .sql("SELECT id FROM vector_search('paimon.default.test_lumina_vector', 'embedding', '[1.0, 0.0, 0.0, 0.0]', 3)") + .sql("SELECT id FROM vector_search('paimon.default.test_java_vindex_vector', 'embedding', '[1.0, 0.0, 0.0, 0.0]', 3)") .await .expect("SQL should parse") .collect() @@ -2275,9 +2247,9 @@ mod vector_search_tests { #[tokio::test] async fn test_vector_search_top6_returns_all() { - let (ctx, _catalog, _tmp) = create_lumina_vector_search_context().await; + let (ctx, _catalog, _tmp) = create_java_vindex_vector_search_context().await; let batches = ctx - .sql("SELECT id FROM vector_search('paimon.default.test_lumina_vector', 'embedding', '[1.0, 0.0, 0.0, 0.0]', 6)") + .sql("SELECT id FROM vector_search('paimon.default.test_java_vindex_vector', 'embedding', '[1.0, 0.0, 0.0, 0.0]', 6)") .await .expect("SQL should parse") .collect() @@ -2290,15 +2262,15 @@ mod vector_search_tests { #[tokio::test] async fn test_vector_search_without_matching_index_returns_empty() { - let (ctx, catalog, _tmp) = create_lumina_vector_search_context().await; + let (ctx, catalog, _tmp) = create_java_vindex_vector_search_context().await; let table = catalog - .get_table(&Identifier::new("default", "test_lumina_vector")) + .get_table(&Identifier::new("default", "test_java_vindex_vector")) .await .expect("load table"); let dropped = table .new_global_index_drop_builder() .with_index_column("embedding") - .with_index_type("lumina") + .with_index_type("ivf-flat") .execute() .await .expect("drop vector indexes"); @@ -2308,7 +2280,7 @@ mod vector_search_tests { ); let batches = ctx - .sql("SELECT id FROM vector_search('paimon.default.test_lumina_vector', 'embedding', '[1.0, 0.0, 0.0, 0.0]', 10)") + .sql("SELECT id FROM vector_search('paimon.default.test_java_vindex_vector', 'embedding', '[1.0, 0.0, 0.0, 0.0]', 10)") .await .expect("SQL should parse") .collect() @@ -2318,15 +2290,15 @@ mod vector_search_tests { let total_rows: usize = batches.iter().map(|batch| batch.num_rows()).sum(); assert_eq!( total_rows, 0, - "vector_search without a matching Lumina index should not fall back to a full table scan" + "vector_search without a matching vindex index should not fall back to a full table scan" ); } #[tokio::test] async fn test_vector_search_rejects_missing_column() { - let (ctx, _catalog, _tmp) = create_lumina_vector_search_context().await; + let (ctx, _catalog, _tmp) = create_java_vindex_vector_search_context().await; let error = ctx - .sql("SELECT id FROM vector_search('paimon.default.test_lumina_vector', 'missing_embedding', '[1.0]', 10)") + .sql("SELECT id FROM vector_search('paimon.default.test_java_vindex_vector', 'missing_embedding', '[1.0]', 10)") .await .expect("SQL should parse") .collect() @@ -2578,17 +2550,12 @@ mod vector_search_tests { ); } - // Manual run with a local Lumina native library: - // LUMINA_LIB_PATH=/path/to/liblumina_py.so cargo test -p paimon-datafusion \ - // vector_search_tests::test_lumina_build_then_vector_search_query \ - // -- --ignored --exact #[tokio::test] - #[ignore = "requires LUMINA_LIB_PATH"] - async fn test_lumina_build_then_vector_search_query() { + async fn test_diskann_build_then_vector_search_query() { let (ctx, catalog, _tmp) = create_empty_vector_search_context().await; - let identifier = Identifier::new("default", "lumina_build_query_e2e"); + let identifier = Identifier::new("default", "diskann_build_query_e2e"); catalog - .create_table(&identifier, build_lumina_table_schema(), false) + .create_table(&identifier, build_vindex_table_schema(), false) .await .expect("Failed to create table"); let table = catalog @@ -2627,15 +2594,15 @@ mod vector_search_tests { .await .expect("Failed to commit vector data"); - ctx.sql("CALL sys.create_lumina_index(table => 'default.lumina_build_query_e2e', index_column => 'embedding')") + ctx.sql("CALL sys.create_global_index(table => 'default.diskann_build_query_e2e', index_column => 'embedding', index_type => 'diskann', options => 'diskann.dimension=2,diskann.distance.metric=cosine,diskann.pq.bits=4,diskann.pq.m=1')") .await - .expect("Lumina index build SQL should parse") + .expect("DiskANN index build SQL should parse") .collect() .await - .expect("Lumina index build SQL should execute"); + .expect("DiskANN index build SQL should execute"); let index_batches = ctx - .sql("SELECT index_type, row_count, row_range_start, row_range_end, index_field_name FROM paimon.default.`lumina_build_query_e2e$table_indexes` WHERE index_type = 'lumina'") + .sql("SELECT index_type, row_count, row_range_start, row_range_end, index_field_name FROM paimon.default.`diskann_build_query_e2e$table_indexes` WHERE index_type = 'diskann'") .await .expect("index metadata SQL should parse") .collect() @@ -2645,13 +2612,13 @@ mod vector_search_tests { assert_eq!( index_rows, vec![ - ("lumina".to_string(), 3, 0, 2, "embedding".to_string()), - ("lumina".to_string(), 3, 3, 5, "embedding".to_string()), + ("diskann".to_string(), 3, 0, 2, "embedding".to_string()), + ("diskann".to_string(), 3, 3, 5, "embedding".to_string()), ] ); let search_batches = ctx - .sql("SELECT id FROM vector_search('paimon.default.lumina_build_query_e2e', 'embedding', '[1.0, 0.0]', 2)") + .sql("SELECT id FROM vector_search('paimon.default.diskann_build_query_e2e', 'embedding', '[1.0, 0.0]', 2)") .await .expect("vector_search SQL should parse") .collect() diff --git a/crates/paimon/src/lib.rs b/crates/paimon/src/lib.rs index ead1c0759..cad8e97f8 100644 --- a/crates/paimon/src/lib.rs +++ b/crates/paimon/src/lib.rs @@ -36,7 +36,6 @@ pub mod ftindex; #[cfg(feature = "fulltext")] pub mod full_text; pub mod io; -pub mod lumina; mod predicate_stats; pub mod resource; pub mod spec; diff --git a/crates/paimon/src/lumina/ffi.rs b/crates/paimon/src/lumina/ffi.rs deleted file mode 100644 index 75b89acb2..000000000 --- a/crates/paimon/src/lumina/ffi.rs +++ /dev/null @@ -1,623 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -use libloading::{Library, Symbol}; -use std::collections::HashMap; -use std::ffi::{c_char, c_float, c_int, c_void, CStr, CString}; -use std::io::{Read, Seek, SeekFrom}; -use std::sync::{Mutex, OnceLock}; - -const ERR_BUF_SIZE: usize = 4096; - -static LIBRARY: OnceLock = OnceLock::new(); -static LIBRARY_LOAD_LOCK: Mutex<()> = Mutex::new(()); - -fn load_library() -> crate::Result<&'static Library> { - if let Some(lib) = LIBRARY.get() { - return Ok(lib); - } - - let _guard = LIBRARY_LOAD_LOCK - .lock() - .map_err(|_| crate::Error::UnexpectedError { - message: "Lumina library load lock poisoned".to_string(), - source: None, - })?; - if let Some(lib) = LIBRARY.get() { - return Ok(lib); - } - - let lib_path = std::env::var("LUMINA_LIB_PATH").unwrap_or_else(|_| { - if cfg!(target_os = "macos") { - "liblumina_py.dylib".to_string() - } else if cfg!(target_os = "windows") { - "lumina_py.dll".to_string() - } else { - "liblumina_py.so".to_string() - } - }); - let lib = unsafe { - Library::new(&lib_path).map_err(|e| crate::Error::DataInvalid { - message: format!("Failed to load lumina library from '{}': {}", lib_path, e), - source: None, - })? - }; - LIBRARY - .set(lib) - .map_err(|_| crate::Error::UnexpectedError { - message: "Lumina library was initialized unexpectedly while holding load lock" - .to_string(), - source: None, - })?; - Ok(LIBRARY - .get() - .expect("Lumina library should be initialized after successful set")) -} - -fn check_error(ret: c_int, err_buf: &[u8; ERR_BUF_SIZE]) -> crate::Result<()> { - if ret != 0 { - let c_str = unsafe { CStr::from_ptr(err_buf.as_ptr() as *const c_char) }; - let msg = c_str.to_string_lossy().to_string(); - return Err(crate::Error::DataInvalid { - message: format!("Lumina error: {}", msg), - source: None, - }); - } - Ok(()) -} - -fn options_to_json(options: &HashMap) -> crate::Result { - let json = serde_json::to_string(options).map_err(|e| crate::Error::DataInvalid { - message: format!("Failed to serialize options: {}", e), - source: None, - })?; - CString::new(json).map_err(|e| crate::Error::DataInvalid { - message: format!("Failed to create CString: {}", e), - source: None, - }) -} - -pub struct LuminaSearcher { - handle: *mut c_void, - /// Keeps the stream context alive while C-side holds a raw pointer to it. - stream_ctx_keepalive: Option>, -} - -// SAFETY: Each LuminaSearcher owns its handle exclusively and is not Sync. -// Send allows moving the searcher to another thread. -unsafe impl Send for LuminaSearcher {} - -impl LuminaSearcher { - pub fn create(options: &HashMap) -> crate::Result { - let lib = load_library()?; - let opts_json = options_to_json(options)?; - let mut err_buf = [0u8; ERR_BUF_SIZE]; - - let handle: *mut c_void = unsafe { - let func: Symbol< - unsafe extern "C" fn(*const c_char, *mut c_char, c_int) -> *mut c_void, - > = lib - .get(b"lumina_searcher_create") - .map_err(|e| crate::Error::DataInvalid { - message: format!("Symbol lumina_searcher_create not found: {}", e), - source: None, - })?; - func( - opts_json.as_ptr(), - err_buf.as_mut_ptr() as *mut c_char, - ERR_BUF_SIZE as c_int, - ) - }; - - if handle.is_null() { - let c_str = unsafe { CStr::from_ptr(err_buf.as_ptr() as *const c_char) }; - let msg = c_str.to_string_lossy().to_string(); - return Err(crate::Error::DataInvalid { - message: format!("Failed to create Lumina searcher: {}", msg), - source: None, - }); - } - - Ok(Self { - handle, - stream_ctx_keepalive: None, - }) - } - - #[allow(clippy::type_complexity)] - pub fn open_stream(&mut self, stream: S) -> crate::Result<()> { - if self.stream_ctx_keepalive.is_some() { - return Err(crate::Error::DataInvalid { - message: "A stream is already open; close the searcher before opening a new stream" - .to_string(), - source: None, - }); - } - - let lib = load_library()?; - let mut err_buf = [0u8; ERR_BUF_SIZE]; - - let ctx = Box::new(StreamContext::new(stream)); - let ctx_ptr = &*ctx as *const StreamContext as *mut c_void; - - let ret: c_int = unsafe { - let func: Symbol< - unsafe extern "C" fn( - *mut c_void, - *mut c_void, - unsafe extern "C" fn(*mut c_void, *mut c_char, u64) -> c_int, - unsafe extern "C" fn(*mut c_void, u64) -> c_int, - unsafe extern "C" fn(*mut c_void) -> u64, - unsafe extern "C" fn(*mut c_void) -> u64, - *mut c_char, - c_int, - ) -> c_int, - > = lib - .get(b"lumina_searcher_open_stream") - .map_err(|e| crate::Error::DataInvalid { - message: format!("Symbol lumina_searcher_open_stream not found: {}", e), - source: None, - })?; - func( - self.handle, - ctx_ptr, - stream_read_cb, - stream_seek_cb, - stream_tell_cb, - stream_length_cb, - err_buf.as_mut_ptr() as *mut c_char, - ERR_BUF_SIZE as c_int, - ) - }; - - check_error(ret, &err_buf)?; - self.stream_ctx_keepalive = Some(ctx); - Ok(()) - } - - pub fn open_file(&mut self, path: &str) -> crate::Result<()> { - if self.stream_ctx_keepalive.is_some() { - return Err(crate::Error::DataInvalid { - message: "A stream is already open; close the searcher before opening a file" - .to_string(), - source: None, - }); - } - - let lib = load_library()?; - let c_path = CString::new(path).map_err(|e| crate::Error::DataInvalid { - message: format!("Invalid path: {}", e), - source: None, - })?; - let mut err_buf = [0u8; ERR_BUF_SIZE]; - - let ret: c_int = unsafe { - let func: Symbol< - unsafe extern "C" fn(*mut c_void, *const c_char, *mut c_char, c_int) -> c_int, - > = lib - .get(b"lumina_searcher_open") - .map_err(|e| crate::Error::DataInvalid { - message: format!("Symbol lumina_searcher_open not found: {}", e), - source: None, - })?; - func( - self.handle, - c_path.as_ptr(), - err_buf.as_mut_ptr() as *mut c_char, - ERR_BUF_SIZE as c_int, - ) - }; - - check_error(ret, &err_buf) - } - - pub fn search( - &self, - query: &[f32], - n: i32, - k: i32, - distances: &mut [f32], - labels: &mut [u64], - options: &HashMap, - ) -> crate::Result<()> { - let lib = load_library()?; - let opts_json = options_to_json(options)?; - let mut err_buf = [0u8; ERR_BUF_SIZE]; - - let ret: c_int = unsafe { - let func: Symbol< - unsafe extern "C" fn( - *mut c_void, - *const c_float, - c_int, - c_int, - *mut c_float, - *mut u64, - *const c_char, - *mut c_char, - c_int, - ) -> c_int, - > = lib - .get(b"lumina_searcher_search") - .map_err(|e| crate::Error::DataInvalid { - message: format!("Symbol lumina_searcher_search not found: {}", e), - source: None, - })?; - func( - self.handle, - query.as_ptr(), - n, - k, - distances.as_mut_ptr(), - labels.as_mut_ptr(), - opts_json.as_ptr(), - err_buf.as_mut_ptr() as *mut c_char, - ERR_BUF_SIZE as c_int, - ) - }; - - check_error(ret, &err_buf) - } - - #[allow(clippy::too_many_arguments, clippy::type_complexity)] - pub fn search_with_filter( - &self, - query: &[f32], - n: i32, - k: i32, - distances: &mut [f32], - labels: &mut [u64], - filter_ids: &[u64], - options: &HashMap, - ) -> crate::Result<()> { - let lib = load_library()?; - let opts_json = options_to_json(options)?; - let mut err_buf = [0u8; ERR_BUF_SIZE]; - - let ret: c_int = unsafe { - let func: Symbol< - unsafe extern "C" fn( - *mut c_void, - *const c_float, - c_int, - c_int, - *mut c_float, - *mut u64, - *const u64, - u64, - *const c_char, - *mut c_char, - c_int, - ) -> c_int, - > = lib - .get(b"lumina_searcher_search_with_filter") - .map_err(|e| crate::Error::DataInvalid { - message: format!("Symbol lumina_searcher_search_with_filter not found: {}", e), - source: None, - })?; - func( - self.handle, - query.as_ptr(), - n, - k, - distances.as_mut_ptr(), - labels.as_mut_ptr(), - filter_ids.as_ptr(), - filter_ids.len() as u64, - opts_json.as_ptr(), - err_buf.as_mut_ptr() as *mut c_char, - ERR_BUF_SIZE as c_int, - ) - }; - - check_error(ret, &err_buf) - } - - pub fn get_count(&self) -> crate::Result { - let lib = load_library()?; - unsafe { - let func: Symbol u64> = lib - .get(b"lumina_searcher_get_count") - .map_err(|e| crate::Error::DataInvalid { - message: format!("Symbol lumina_searcher_get_count not found: {}", e), - source: None, - })?; - Ok(func(self.handle)) - } - } - - pub fn get_dimension(&self) -> crate::Result { - let lib = load_library()?; - unsafe { - let func: Symbol u32> = lib - .get(b"lumina_searcher_get_dimension") - .map_err(|e| crate::Error::DataInvalid { - message: format!("Symbol lumina_searcher_get_dimension not found: {}", e), - source: None, - })?; - Ok(func(self.handle)) - } - } -} - -impl Drop for LuminaSearcher { - fn drop(&mut self) { - if !self.handle.is_null() { - if let Ok(lib) = load_library() { - unsafe { - if let Ok(func) = - lib.get::(b"lumina_searcher_destroy") - { - func(self.handle); - } - } - } - self.handle = std::ptr::null_mut(); - } - } -} - -pub struct LuminaBuilder { - handle: *mut c_void, -} - -// SAFETY: Same as LuminaSearcher — exclusively owned, not Sync. -unsafe impl Send for LuminaBuilder {} - -impl LuminaBuilder { - pub fn create(options: &HashMap) -> crate::Result { - let lib = load_library()?; - let opts_json = options_to_json(options)?; - let mut err_buf = [0u8; ERR_BUF_SIZE]; - - let handle: *mut c_void = unsafe { - let func: Symbol< - unsafe extern "C" fn(*const c_char, *mut c_char, c_int) -> *mut c_void, - > = lib - .get(b"lumina_builder_create") - .map_err(|e| crate::Error::DataInvalid { - message: format!("Symbol lumina_builder_create not found: {}", e), - source: None, - })?; - func( - opts_json.as_ptr(), - err_buf.as_mut_ptr() as *mut c_char, - ERR_BUF_SIZE as c_int, - ) - }; - - if handle.is_null() { - let c_str = unsafe { CStr::from_ptr(err_buf.as_ptr() as *const c_char) }; - let msg = c_str.to_string_lossy().to_string(); - return Err(crate::Error::DataInvalid { - message: format!("Failed to create Lumina builder: {}", msg), - source: None, - }); - } - - Ok(Self { handle }) - } - - pub fn pretrain(&self, vectors: &[f32], n: i32, dim: i32) -> crate::Result<()> { - let lib = load_library()?; - let mut err_buf = [0u8; ERR_BUF_SIZE]; - - let ret: c_int = unsafe { - let func: Symbol< - unsafe extern "C" fn( - *mut c_void, - *const c_float, - c_int, - c_int, - *mut c_char, - c_int, - ) -> c_int, - > = lib - .get(b"lumina_builder_pretrain") - .map_err(|e| crate::Error::DataInvalid { - message: format!("Symbol lumina_builder_pretrain not found: {}", e), - source: None, - })?; - func( - self.handle, - vectors.as_ptr(), - n, - dim, - err_buf.as_mut_ptr() as *mut c_char, - ERR_BUF_SIZE as c_int, - ) - }; - - check_error(ret, &err_buf) - } - - pub fn insert(&self, vectors: &[f32], ids: &[u64], n: i32, dim: i32) -> crate::Result<()> { - let lib = load_library()?; - let mut err_buf = [0u8; ERR_BUF_SIZE]; - - let ret: c_int = unsafe { - let func: Symbol< - unsafe extern "C" fn( - *mut c_void, - *const c_float, - *const u64, - c_int, - c_int, - *mut c_char, - c_int, - ) -> c_int, - > = lib - .get(b"lumina_builder_insert") - .map_err(|e| crate::Error::DataInvalid { - message: format!("Symbol lumina_builder_insert not found: {}", e), - source: None, - })?; - func( - self.handle, - vectors.as_ptr(), - ids.as_ptr(), - n, - dim, - err_buf.as_mut_ptr() as *mut c_char, - ERR_BUF_SIZE as c_int, - ) - }; - - check_error(ret, &err_buf) - } - - pub fn dump(&self, path: &str) -> crate::Result<()> { - let lib = load_library()?; - let c_path = CString::new(path).map_err(|e| crate::Error::DataInvalid { - message: format!("Invalid path: {}", e), - source: None, - })?; - let mut err_buf = [0u8; ERR_BUF_SIZE]; - - let ret: c_int = unsafe { - let func: Symbol< - unsafe extern "C" fn(*mut c_void, *const c_char, *mut c_char, c_int) -> c_int, - > = lib - .get(b"lumina_builder_dump") - .map_err(|e| crate::Error::DataInvalid { - message: format!("Symbol lumina_builder_dump not found: {}", e), - source: None, - })?; - func( - self.handle, - c_path.as_ptr(), - err_buf.as_mut_ptr() as *mut c_char, - ERR_BUF_SIZE as c_int, - ) - }; - - check_error(ret, &err_buf) - } -} - -impl Drop for LuminaBuilder { - fn drop(&mut self) { - if !self.handle.is_null() { - if let Ok(lib) = load_library() { - unsafe { - if let Ok(func) = - lib.get::(b"lumina_builder_destroy") - { - func(self.handle); - } - } - } - self.handle = std::ptr::null_mut(); - } - } -} - -struct StreamContext { - inner: std::sync::Mutex>, -} - -trait ReadSeekLen: Read + Seek { - fn length(&self) -> u64; -} - -struct ReadSeekLenImpl { - stream: S, - len: u64, -} - -impl ReadSeekLenImpl { - fn new(mut stream: S) -> Self { - let len = stream.seek(SeekFrom::End(0)).unwrap_or(0); - let _ = stream.seek(SeekFrom::Start(0)); - Self { stream, len } - } -} - -impl Read for ReadSeekLenImpl { - fn read(&mut self, buf: &mut [u8]) -> std::io::Result { - self.stream.read(buf) - } -} - -impl Seek for ReadSeekLenImpl { - fn seek(&mut self, pos: SeekFrom) -> std::io::Result { - self.stream.seek(pos) - } -} - -impl ReadSeekLen for ReadSeekLenImpl { - fn length(&self) -> u64 { - self.len - } -} - -impl StreamContext { - fn new(stream: S) -> Self { - Self { - inner: std::sync::Mutex::new(Box::new(ReadSeekLenImpl::new(stream))), - } - } -} - -unsafe extern "C" fn stream_read_cb(ctx: *mut c_void, buf: *mut c_char, size: u64) -> c_int { - let ctx = &*(ctx as *const StreamContext); - let mut guard = match ctx.inner.lock() { - Ok(g) => g, - Err(_) => return -1, - }; - let clamped_size = std::cmp::min(size, c_int::MAX as u64) as usize; - let slice = std::slice::from_raw_parts_mut(buf as *mut u8, clamped_size); - let mut total_read = 0usize; - while total_read < clamped_size { - match guard.read(&mut slice[total_read..]) { - Ok(0) => break, - Ok(n) => total_read += n, - Err(_) => return -1, - } - } - std::cmp::min(total_read, c_int::MAX as usize) as c_int -} - -unsafe extern "C" fn stream_seek_cb(ctx: *mut c_void, position: u64) -> c_int { - let ctx = &*(ctx as *const StreamContext); - let mut guard = match ctx.inner.lock() { - Ok(g) => g, - Err(_) => return -1, - }; - match guard.seek(SeekFrom::Start(position)) { - Ok(_) => 0, - Err(_) => -1, - } -} - -unsafe extern "C" fn stream_tell_cb(ctx: *mut c_void) -> u64 { - let ctx = &*(ctx as *const StreamContext); - let mut guard = match ctx.inner.lock() { - Ok(g) => g, - Err(_) => return 0, - }; - guard.seek(SeekFrom::Current(0)).unwrap_or(0) -} - -unsafe extern "C" fn stream_length_cb(ctx: *mut c_void) -> u64 { - let ctx = &*(ctx as *const StreamContext); - let guard = match ctx.inner.lock() { - Ok(g) => g, - Err(_) => return 0, - }; - guard.length() -} diff --git a/crates/paimon/src/lumina/mod.rs b/crates/paimon/src/lumina/mod.rs deleted file mode 100644 index 2302fd9f2..000000000 --- a/crates/paimon/src/lumina/mod.rs +++ /dev/null @@ -1,579 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -pub mod ffi; -pub mod reader; - -use std::collections::HashMap; - -pub const LUMINA_IDENTIFIER: &str = "lumina"; -pub const LEGACY_LUMINA_VECTOR_ANN_IDENTIFIER: &str = "lumina-vector-ann"; -pub const LUMINA_VECTOR_ANN_IDENTIFIER: &str = LEGACY_LUMINA_VECTOR_ANN_IDENTIFIER; - -pub fn is_lumina_index_type(index_type: &str) -> bool { - matches!( - index_type, - LUMINA_IDENTIFIER | LEGACY_LUMINA_VECTOR_ANN_IDENTIFIER - ) -} - -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub enum LuminaVectorMetric { - L2, - Cosine, - InnerProduct, -} - -impl LuminaVectorMetric { - pub fn lumina_name(&self) -> &str { - match self { - LuminaVectorMetric::L2 => "l2", - LuminaVectorMetric::Cosine => "cosine", - LuminaVectorMetric::InnerProduct => "inner_product", - } - } - - pub fn from_string(name: &str) -> crate::Result { - match name.to_uppercase().as_str() { - "L2" => Ok(LuminaVectorMetric::L2), - "COSINE" => Ok(LuminaVectorMetric::Cosine), - "INNER_PRODUCT" => Ok(LuminaVectorMetric::InnerProduct), - _ => Err(crate::Error::DataInvalid { - message: format!("Unknown metric name: {}", name), - source: None, - }), - } - } - - pub fn from_lumina_name(lumina_name: &str) -> crate::Result { - match lumina_name { - "l2" => Ok(LuminaVectorMetric::L2), - "cosine" => Ok(LuminaVectorMetric::Cosine), - "inner_product" => Ok(LuminaVectorMetric::InnerProduct), - _ => Err(crate::Error::DataInvalid { - message: format!("Unknown lumina metric name: {}", lumina_name), - source: None, - }), - } - } -} - -const LUMINA_PREFIX: &str = "lumina."; - -const ALL_OPTIONS_DEFAULTS: &[(&str, &str)] = &[ - ("lumina.index.dimension", "128"), - ("lumina.index.type", "diskann"), - ("lumina.distance.metric", "inner_product"), - ("lumina.encoding.type", "pq"), - ("lumina.pretrain.sample_ratio", "0.2"), - ("lumina.diskann.build.ef_construction", "1024"), - ("lumina.diskann.build.neighbor_count", "64"), - ("lumina.diskann.build.thread_count", "32"), - ("lumina.diskann.search.beam_width", "4"), - ("lumina.encoding.pq.m", "64"), - ("lumina.search.parallel_number", "5"), -]; - -pub struct LuminaVectorIndexOptions { - pub dimension: i32, - pub metric: LuminaVectorMetric, - pub index_type: String, - lumina_options: HashMap, -} - -impl LuminaVectorIndexOptions { - pub fn new(paimon_options: &HashMap) -> crate::Result { - let dimension_str = paimon_options - .get("lumina.index.dimension") - .map(|s| s.as_str()) - .unwrap_or("128"); - let dimension: i32 = dimension_str - .parse() - .map_err(|_| crate::Error::DataInvalid { - message: format!("Invalid dimension: {}", dimension_str), - source: None, - })?; - if dimension <= 0 { - return Err(crate::Error::DataInvalid { - message: format!( - "Invalid value for 'lumina.index.dimension': {}. Must be a positive integer.", - dimension - ), - source: None, - }); - } - - let metric_str = paimon_options - .get("lumina.distance.metric") - .map(|s| s.as_str()) - .unwrap_or("inner_product"); - let metric = LuminaVectorMetric::from_lumina_name(metric_str) - .or_else(|_| LuminaVectorMetric::from_string(metric_str))?; - - let encoding = paimon_options - .get("lumina.encoding.type") - .map(|s| s.as_str()) - .unwrap_or("pq"); - validate_encoding_metric(encoding, metric)?; - - let index_type = paimon_options - .get("lumina.index.type") - .cloned() - .unwrap_or_else(|| "diskann".to_string()); - - let mut lumina_options = build_lumina_options(paimon_options, dimension)?; - canonicalize_metric(&mut lumina_options); - - Ok(Self { - dimension, - metric, - index_type, - lumina_options, - }) - } - - pub fn to_lumina_options(&self) -> HashMap { - self.lumina_options.clone() - } -} - -/// Rewrite `distance.metric` to the spelling `from_lumina_name` accepts. -/// -/// The option parser also takes the enum names, via `from_string` above, but -/// `LuminaIndexMeta::metric` is exact-match, so the configured spelling must not -/// reach the committed metadata. An unrecognized value is left alone, keeping its -/// existing error surface. -fn canonicalize_metric(native_options: &mut HashMap) { - let Some(configured) = native_options.get(KEY_DISTANCE_METRIC) else { - return; - }; - let parsed = LuminaVectorMetric::from_lumina_name(configured) - .or_else(|_| LuminaVectorMetric::from_string(configured)); - if let Ok(metric) = parsed { - native_options.insert( - KEY_DISTANCE_METRIC.to_string(), - metric.lumina_name().to_string(), - ); - } -} - -fn validate_encoding_metric(encoding: &str, metric: LuminaVectorMetric) -> crate::Result<()> { - if encoding.eq_ignore_ascii_case("pq") && metric == LuminaVectorMetric::Cosine { - return Err(crate::Error::DataInvalid { - message: - "Lumina does not support PQ encoding with cosine metric. \ - Please use 'rawf32' or 'sq8' encoding, or switch to 'l2' or 'inner_product' metric." - .to_string(), - source: None, - }); - } - Ok(()) -} - -fn validate_and_cap_pq_m(opts: &mut HashMap, dimension: i32) -> crate::Result<()> { - let encoding = opts.get("encoding.type").map(|s| s.as_str()).unwrap_or(""); - if !encoding.eq_ignore_ascii_case("pq") { - return Ok(()); - } - if let Some(pq_m_str) = opts.get("encoding.pq.m") { - let pq_m: i32 = pq_m_str.parse().map_err(|_| crate::Error::DataInvalid { - message: format!("encoding.pq.m must be an integer, got: {}", pq_m_str), - source: None, - })?; - if pq_m <= 0 { - return Err(crate::Error::DataInvalid { - message: format!("encoding.pq.m must be positive, got: {}", pq_m), - source: None, - }); - } - if pq_m > dimension { - opts.insert("encoding.pq.m".to_string(), dimension.to_string()); - } - } - Ok(()) -} - -fn build_lumina_options( - paimon_options: &HashMap, - dimension: i32, -) -> crate::Result> { - let mut result = HashMap::new(); - - for &(paimon_key, default_value) in ALL_OPTIONS_DEFAULTS { - let native_key = &paimon_key[LUMINA_PREFIX.len()..]; - let value = paimon_options - .get(paimon_key) - .map(|s| s.as_str()) - .unwrap_or(default_value); - result.insert(native_key.to_string(), value.to_string()); - } - - for (key, value) in paimon_options { - if key == LUMINA_SEARCH_MAX_FILTER_BYTES_OPTION { - continue; - } - if let Some(native_key) = key.strip_prefix(LUMINA_PREFIX) { - result - .entry(native_key.to_string()) - .or_insert_with(|| value.to_string()); - } - } - - validate_and_cap_pq_m(&mut result, dimension)?; - Ok(result) -} - -pub fn strip_lumina_options(paimon_options: &HashMap) -> HashMap { - let mut result = HashMap::new(); - for (key, value) in paimon_options { - if key == LUMINA_SEARCH_MAX_FILTER_BYTES_OPTION { - continue; - } - if let Some(native_key) = key.strip_prefix(LUMINA_PREFIX) { - result.insert(native_key.to_string(), value.to_string()); - } - } - // `LuminaVectorReader::ensure_loaded` overlays this map on top of the index - // metadata, so a table option spelled `L2` would put the enum name back. - canonicalize_metric(&mut result); - result -} - -pub const KEY_DIMENSION: &str = "index.dimension"; -pub const KEY_DISTANCE_METRIC: &str = "distance.metric"; -pub const KEY_INDEX_TYPE: &str = "index.type"; - -/// Paimon-prefixed option key for the vector index dimension. -pub const LUMINA_DIMENSION_OPTION: &str = "lumina.index.dimension"; -/// Rust reader memory budget for densifying a Lumina row-id filter. -pub const LUMINA_SEARCH_MAX_FILTER_BYTES_OPTION: &str = "lumina.search.max-filter-bytes"; - -pub struct LuminaIndexMeta { - options: HashMap, -} - -impl LuminaIndexMeta { - pub fn new(options: HashMap) -> Self { - Self { options } - } - - pub fn options(&self) -> &HashMap { - &self.options - } - - pub fn dim(&self) -> crate::Result { - let val = self - .options - .get(KEY_DIMENSION) - .ok_or_else(|| crate::Error::DataInvalid { - message: format!("Missing required key: {}", KEY_DIMENSION), - source: None, - })?; - val.parse::().map_err(|_| crate::Error::DataInvalid { - message: format!("Invalid dimension value: {}", val), - source: None, - }) - } - - pub fn distance_metric(&self) -> &str { - self.options - .get(KEY_DISTANCE_METRIC) - .map(String::as_str) - .unwrap_or("") - } - - pub fn metric(&self) -> crate::Result { - LuminaVectorMetric::from_lumina_name(self.distance_metric()) - } - - pub fn index_type(&self) -> &str { - self.options - .get(KEY_INDEX_TYPE) - .map(String::as_str) - .unwrap_or("diskann") - } - - pub fn serialize(&self) -> crate::Result> { - serde_json::to_vec(&self.options).map_err(|e| crate::Error::DataInvalid { - message: format!("Failed to serialize LuminaIndexMeta: {}", e), - source: None, - }) - } - - pub fn deserialize(data: &[u8]) -> crate::Result { - let options: HashMap = - serde_json::from_slice(data).map_err(|e| crate::Error::DataInvalid { - message: format!("Failed to deserialize LuminaIndexMeta: {}", e), - source: None, - })?; - if !options.contains_key(KEY_DIMENSION) { - return Err(crate::Error::DataInvalid { - message: format!( - "Missing required key in Lumina index metadata: {}", - KEY_DIMENSION - ), - source: None, - }); - } - if !options.contains_key(KEY_DISTANCE_METRIC) { - return Err(crate::Error::DataInvalid { - message: format!( - "Missing required key in Lumina index metadata: {}", - KEY_DISTANCE_METRIC - ), - source: None, - }); - } - Ok(Self { options }) - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn test_metric_roundtrip() { - for metric in [ - LuminaVectorMetric::L2, - LuminaVectorMetric::Cosine, - LuminaVectorMetric::InnerProduct, - ] { - let name = metric.lumina_name(); - assert_eq!(LuminaVectorMetric::from_lumina_name(name).unwrap(), metric); - assert_eq!( - LuminaVectorMetric::from_string(&name.to_uppercase()).unwrap(), - metric - ); - } - assert!(LuminaVectorMetric::from_string("hamming").is_err()); - } - - #[test] - fn test_lumina_index_type_identifier_helper() { - assert!(is_lumina_index_type(LUMINA_IDENTIFIER)); - assert!(is_lumina_index_type(LEGACY_LUMINA_VECTOR_ANN_IDENTIFIER)); - assert!(is_lumina_index_type(LUMINA_VECTOR_ANN_IDENTIFIER)); - assert!(!is_lumina_index_type("")); - assert!(!is_lumina_index_type("btree")); - assert!(!is_lumina_index_type("lumina-vector")); - assert!(!is_lumina_index_type("LUMINA")); - } - - #[test] - fn test_index_meta_serialize_deserialize() { - let mut options = HashMap::new(); - options.insert(KEY_DIMENSION.to_string(), "128".to_string()); - options.insert(KEY_DISTANCE_METRIC.to_string(), "l2".to_string()); - options.insert(KEY_INDEX_TYPE.to_string(), "diskann".to_string()); - let meta = LuminaIndexMeta::new(options); - - let bytes = meta.serialize().unwrap(); - let meta2 = LuminaIndexMeta::deserialize(&bytes).unwrap(); - assert_eq!(meta2.dim().unwrap(), 128); - assert_eq!(meta2.distance_metric(), "l2"); - assert_eq!(meta2.index_type(), "diskann"); - } - - #[test] - fn test_index_meta_deserialize_missing_fields() { - // missing dimension - let mut opts = HashMap::new(); - opts.insert(KEY_DISTANCE_METRIC.to_string(), "l2".to_string()); - assert!(LuminaIndexMeta::deserialize(&serde_json::to_vec(&opts).unwrap()).is_err()); - - // missing metric - let mut opts = HashMap::new(); - opts.insert(KEY_DIMENSION.to_string(), "128".to_string()); - assert!(LuminaIndexMeta::deserialize(&serde_json::to_vec(&opts).unwrap()).is_err()); - - // invalid json - assert!(LuminaIndexMeta::deserialize(b"not json").is_err()); - } - - #[test] - fn test_dim_error_on_invalid() { - let mut opts = HashMap::new(); - opts.insert(KEY_DIMENSION.to_string(), "abc".to_string()); - opts.insert(KEY_DISTANCE_METRIC.to_string(), "l2".to_string()); - assert!(LuminaIndexMeta::new(opts).dim().is_err()); - } - - #[test] - fn test_index_options_invalid_dimension() { - let mut opts = HashMap::new(); - opts.insert("lumina.index.dimension".to_string(), "-1".to_string()); - assert!(LuminaVectorIndexOptions::new(&opts).is_err()); - } - - #[test] - fn test_strip_lumina_options() { - let mut opts = HashMap::new(); - opts.insert("lumina.index.dimension".to_string(), "128".to_string()); - opts.insert( - "lumina.diskann.search.beam_width".to_string(), - "8".to_string(), - ); - opts.insert( - LUMINA_SEARCH_MAX_FILTER_BYTES_OPTION.to_string(), - "64 mb".to_string(), - ); - opts.insert("non_lumina_key".to_string(), "ignored".to_string()); - let result = strip_lumina_options(&opts); - assert_eq!(result.get("index.dimension").unwrap(), "128"); - assert_eq!(result.get("diskann.search.beam_width").unwrap(), "8"); - assert!(!result.contains_key("search.max-filter-bytes")); - assert!(!result.contains_key("non_lumina_key")); - } - - #[test] - fn rust_dense_filter_budget_is_not_a_native_build_option() { - let options = HashMap::from([( - LUMINA_SEARCH_MAX_FILTER_BYTES_OPTION.to_string(), - "64 mb".to_string(), - )]); - let native = LuminaVectorIndexOptions::new(&options) - .unwrap() - .to_lumina_options(); - assert!(!native.contains_key("search.max-filter-bytes")); - } - - #[test] - fn test_pq_cosine_rejected() { - let mut opts = HashMap::new(); - opts.insert("lumina.index.dimension".to_string(), "128".to_string()); - opts.insert("lumina.distance.metric".to_string(), "cosine".to_string()); - opts.insert("lumina.encoding.type".to_string(), "pq".to_string()); - assert!(LuminaVectorIndexOptions::new(&opts).is_err()); - } - - #[test] - fn test_pq_l2_accepted() { - let mut opts = HashMap::new(); - opts.insert("lumina.index.dimension".to_string(), "128".to_string()); - opts.insert("lumina.distance.metric".to_string(), "l2".to_string()); - opts.insert("lumina.encoding.type".to_string(), "pq".to_string()); - assert!(LuminaVectorIndexOptions::new(&opts).is_ok()); - } - - #[test] - fn test_pq_m_zero_rejected() { - let mut opts = HashMap::new(); - opts.insert("lumina.index.dimension".to_string(), "128".to_string()); - opts.insert("lumina.encoding.pq.m".to_string(), "0".to_string()); - assert!(LuminaVectorIndexOptions::new(&opts).is_err()); - } - - #[test] - fn test_pq_m_non_numeric_rejected() { - let mut opts = HashMap::new(); - opts.insert("lumina.index.dimension".to_string(), "128".to_string()); - opts.insert("lumina.encoding.pq.m".to_string(), "abc".to_string()); - assert!(LuminaVectorIndexOptions::new(&opts).is_err()); - } - - #[test] - fn test_cap_pq_m() { - let mut opts = HashMap::new(); - opts.insert("lumina.index.dimension".to_string(), "32".to_string()); - opts.insert("lumina.encoding.pq.m".to_string(), "64".to_string()); - let index_opts = LuminaVectorIndexOptions::new(&opts).unwrap(); - let lumina_opts = index_opts.to_lumina_options(); - assert_eq!(lumina_opts.get("encoding.pq.m").unwrap(), "32"); - } - - #[test] - fn test_build_lumina_options_defaults() { - let opts = HashMap::new(); - let index_opts = LuminaVectorIndexOptions::new(&opts).unwrap(); - let lumina_opts = index_opts.to_lumina_options(); - assert_eq!(lumina_opts.get("index.dimension").unwrap(), "128"); - assert_eq!(lumina_opts.get("distance.metric").unwrap(), "inner_product"); - assert_eq!(lumina_opts.get("encoding.type").unwrap(), "pq"); - assert_eq!(lumina_opts.get("pretrain.sample_ratio").unwrap(), "0.2"); - assert_eq!( - lumina_opts.get("diskann.build.ef_construction").unwrap(), - "1024" - ); - assert_eq!( - lumina_opts.get("diskann.build.neighbor_count").unwrap(), - "64" - ); - assert_eq!(lumina_opts.get("diskann.build.thread_count").unwrap(), "32"); - assert_eq!(lumina_opts.get("diskann.search.beam_width").unwrap(), "4"); - assert_eq!(lumina_opts.get("encoding.pq.m").unwrap(), "64"); - assert_eq!(lumina_opts.get("search.parallel_number").unwrap(), "5"); - } - - #[test] - fn test_enum_form_metric_is_canonicalized_in_native_options() { - // The native-spelling rows pin that canonicalization is a no-op for them. - let cases = [ - ("L2", "l2"), - ("COSINE", "cosine"), - ("INNER_PRODUCT", "inner_product"), - ("l2", "l2"), - ("cosine", "cosine"), - ("inner_product", "inner_product"), - ]; - for (configured, native) in cases { - let paimon_options = HashMap::from([ - ("lumina.index.dimension".to_string(), "4".to_string()), - // `cosine` is rejected with the default `pq` encoding. - ("lumina.encoding.type".to_string(), "rawf32".to_string()), - ("lumina.distance.metric".to_string(), configured.to_string()), - ]); - let options = LuminaVectorIndexOptions::new(&paimon_options).unwrap(); - let native_options = options.to_lumina_options(); - assert_eq!( - native_options.get(KEY_DISTANCE_METRIC).map(String::as_str), - Some(native), - "configured: {configured}" - ); - let metric = LuminaIndexMeta::new(native_options) - .metric() - .unwrap_or_else(|e| panic!("configured {configured}: {e}")); - assert_eq!(metric.lumina_name(), native, "configured: {configured}"); - } - } - - #[test] - fn test_stripped_search_options_use_the_canonical_metric_name() { - let stripped = strip_lumina_options(&HashMap::from([( - "lumina.distance.metric".to_string(), - "L2".to_string(), - )])); - assert_eq!( - stripped.get(KEY_DISTANCE_METRIC).map(String::as_str), - Some("l2") - ); - } - - /// An unknown metric keeps its existing error surface rather than being - /// defaulted here. Passes without the fix too. - #[test] - fn test_stripped_search_options_pass_an_unknown_metric_through() { - let stripped = strip_lumina_options(&HashMap::from([( - "lumina.distance.metric".to_string(), - "hamming".to_string(), - )])); - assert_eq!( - stripped.get(KEY_DISTANCE_METRIC).map(String::as_str), - Some("hamming") - ); - } -} diff --git a/crates/paimon/src/lumina/reader.rs b/crates/paimon/src/lumina/reader.rs deleted file mode 100644 index 90640cf1b..000000000 --- a/crates/paimon/src/lumina/reader.rs +++ /dev/null @@ -1,1521 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -use crate::lumina::ffi::LuminaSearcher; -use crate::lumina::{ - strip_lumina_options, LuminaIndexMeta, LuminaVectorMetric, - LUMINA_SEARCH_MAX_FILTER_BYTES_OPTION, -}; -use crate::vector_search::{GlobalIndexIOMeta, VectorSearch}; -use std::collections::BinaryHeap; -use std::collections::HashMap; -use std::io::{Read, Seek, SeekFrom}; -use std::path::PathBuf; - -const MIN_SEARCH_LIST_SIZE: usize = 16; -// Per active Lumina reader. This admits 8,388,608 row ids while bounding -// amplification from the default 32-way global-index search concurrency to about -// 2 GiB of dense-filter storage. Larger segments can override the option explicitly. -const DEFAULT_LUMINA_SEARCH_MAX_FILTER_BYTES: usize = 64 * 1024 * 1024; -// C ABI returns int64_t -1 for invalid results, which casts to u64::MAX in Rust. -const SENTINEL: u64 = u64::MAX; - -trait LuminaSearch { - fn search( - &self, - query: &[f32], - n: i32, - k: i32, - distances: &mut [f32], - labels: &mut [u64], - options: &HashMap, - ) -> crate::Result<()>; - - #[allow(clippy::too_many_arguments)] - fn search_with_filter( - &self, - query: &[f32], - n: i32, - k: i32, - distances: &mut [f32], - labels: &mut [u64], - filter_ids: &[u64], - options: &HashMap, - ) -> crate::Result<()>; - - fn get_count(&self) -> crate::Result; -} - -impl LuminaSearch for LuminaSearcher { - fn search( - &self, - query: &[f32], - n: i32, - k: i32, - distances: &mut [f32], - labels: &mut [u64], - options: &HashMap, - ) -> crate::Result<()> { - LuminaSearcher::search(self, query, n, k, distances, labels, options) - } - - fn search_with_filter( - &self, - query: &[f32], - n: i32, - k: i32, - distances: &mut [f32], - labels: &mut [u64], - filter_ids: &[u64], - options: &HashMap, - ) -> crate::Result<()> { - LuminaSearcher::search_with_filter( - self, query, n, k, distances, labels, filter_ids, options, - ) - } - - fn get_count(&self) -> crate::Result { - LuminaSearcher::get_count(self) - } -} - -fn ensure_search_list_size(search_options: &mut HashMap, top_k: usize) { - if !search_options.contains_key("diskann.search.list_size") { - let list_size = std::cmp::max((top_k as f64 * 1.5) as usize, MIN_SEARCH_LIST_SIZE); - search_options.insert( - "diskann.search.list_size".to_string(), - list_size.to_string(), - ); - } -} - -/// Merge the table's Lumina options with the options recorded in the index. -/// -/// Java's order at the searcher-open site is the table's options first, then -/// `indexMeta.options()` on top (`LuminaVectorGlobalIndexReader:377-378`), so the -/// index's own geometry wins. Per-query options are layered on top of that base at -/// each search instead, in [`search_options_for_query`], reproducing Java's -/// base -> indexMeta -> queryOptions order (`:253-255`). -/// -/// The order is not cosmetic: the build side deliberately rewrites some of these keys -/// before the index is written. `validate_and_cap_pq_m` caps `encoding.pq.m` at the -/// column's dimension, and the capped value is what the index file was actually built -/// with. Letting the table's raw value win would open the searcher with a PQ geometry -/// the index does not have. `LuminaSearcher::create` takes only this map -- unlike -/// Java's `LuminaIndex.fromStream`, which is also handed `indexMeta.dim()` and -/// `indexMeta.metric()` as separate arguments -- so here the map is the sole channel. -/// -/// `build_lumina_options` seeds every key of `ALL_OPTIONS_DEFAULTS` into the metadata, -/// so a Rust-built index records the build-time `diskann.search.beam_width` and -/// `search.parallel_number` too, and those now win over a table option, as they do in -/// Java. It also folds in any other `lumina.*` key present at build time, so what the -/// table layer can still supply is whatever was not set when the index was built. A -/// query overrides any of it, in [`search_options_for_query`]. -fn merge_searcher_options( - table_options: &HashMap, - index_meta: &LuminaIndexMeta, -) -> HashMap { - let mut merged = strip_lumina_options(table_options); - for (key, value) in index_meta.options() { - merged.insert(key.clone(), value.clone()); - } - merged -} - -/// Layer one query's own options over the searcher's base map, last, as Java's -/// `buildSearchOptions` does (`LuminaVectorGlobalIndexReader:253-255`). -/// -/// Both read paths fold the per-query map into the table options before building this -/// reader (`de_vector_read.rs`, `pk_vector_read.rs`), so without this layer the index -/// metadata would also win over a per-query override -- and for a Rust-built index it -/// always would, since the metadata records every `ALL_OPTIONS_DEFAULTS` key. -fn search_options_for_query( - search_options_base: &HashMap, - vector_search: &VectorSearch, -) -> HashMap { - let mut search_opts = search_options_base.clone(); - for (key, value) in strip_lumina_options(&vector_search.options) { - search_opts.insert(key, value); - } - search_opts -} - -fn convert_distance_to_score(distance: f32, metric: LuminaVectorMetric) -> f32 { - match metric { - LuminaVectorMetric::L2 => 1.0 / (1.0 + distance), - LuminaVectorMetric::Cosine => 1.0 - distance, - LuminaVectorMetric::InnerProduct => distance, - } -} - -/// Order two search scores, best last, with NaN ranked below every real score. -/// -/// This is the score-domain mirror of `vindex::pkvector::metric`'s -/// `java_float_compare`, which ranks a NaN *distance* worst. `f32::total_cmp` -/// alone is unsuitable in either domain: it places a positive NaN above every -/// finite value, so a NaN score -- reachable here because a non-finite stored -/// vector yields a NaN distance and [`convert_distance_to_score`] passes NaN -/// through -- would outrank real neighbours. Both NaN signs lose, and two NaNs -/// compare equal so the caller's row-id tie-break decides between them. -fn compare_scores(a: f32, b: f32) -> std::cmp::Ordering { - match (a.is_nan(), b.is_nan()) { - (true, true) => std::cmp::Ordering::Equal, - (true, false) => std::cmp::Ordering::Less, - (false, true) => std::cmp::Ordering::Greater, - (false, false) => a.total_cmp(&b), - } -} - -/// Turn an include-set into the dense id array Lumina's filtered search takes. -/// -/// Java's `Integer.MAX_VALUE` bound is retained, and the Rust reader adds a byte -/// budget because Lumina alone expands the compressed set into `Vec`. -fn to_scoped_ids( - include_row_ids: &roaring::RoaringTreemap, - max_filter_bytes: usize, -) -> crate::Result> { - let cardinality = include_row_ids.len(); - if cardinality > i32::MAX as u64 { - return Err(crate::Error::DataInvalid { - message: format!( - "include_row_ids cardinality ({cardinality}) exceeds {}", - i32::MAX - ), - source: None, - }); - } - let cardinality = cardinality as usize; - let required_bytes = cardinality - .checked_mul(std::mem::size_of::()) - .ok_or_else(|| crate::Error::DataInvalid { - message: "Lumina dense row-id filter size overflow".to_string(), - source: None, - })?; - if required_bytes > max_filter_bytes { - return Err(crate::Error::DataInvalid { - message: format!( - "Lumina dense row-id filter requires {required_bytes} bytes, exceeding \ - {LUMINA_SEARCH_MAX_FILTER_BYTES_OPTION}={max_filter_bytes}" - ), - source: None, - }); - } - let mut ids = Vec::new(); - ids.try_reserve_exact(cardinality) - .map_err(|error| crate::Error::UnexpectedError { - message: format!( - "failed to allocate {required_bytes} bytes for Lumina dense row-id filter" - ), - source: Some(Box::new(error)), - })?; - ids.extend(include_row_ids.iter()); - Ok(ids) -} - -fn max_filter_bytes(options: &HashMap) -> crate::Result { - let Some(value) = options.get(LUMINA_SEARCH_MAX_FILTER_BYTES_OPTION) else { - return Ok(DEFAULT_LUMINA_SEARCH_MAX_FILTER_BYTES); - }; - let bytes = crate::common::options::parse_memory_size(value).map_err(|_| { - crate::Error::ConfigInvalid { - message: format!( - "Invalid {LUMINA_SEARCH_MAX_FILTER_BYTES_OPTION}: '{value}'; expected a positive memory size" - ), - } - })?; - if bytes <= 0 { - return Err(crate::Error::ConfigInvalid { - message: format!( - "{LUMINA_SEARCH_MAX_FILTER_BYTES_OPTION} must be greater than 0, got: {value}" - ), - }); - } - usize::try_from(bytes).map_err(|_| crate::Error::ConfigInvalid { - message: format!( - "{LUMINA_SEARCH_MAX_FILTER_BYTES_OPTION} does not fit this platform: {value}" - ), - }) -} - -/// Allocate the label buffer for one native search, filled with [`SENTINEL`]. -/// -/// [`SENTINEL`] is the "no result" marker (the C ABI's `-1`), and -/// [`collect_results`] drops any slot carrying it. Zero-filling the buffer -/// instead makes that marker unreliable: `0` is a *legal* row id, so a slot the -/// searcher leaves untouched is indistinguishable from a real hit, and it pairs -/// with distance `0.0` -- the best distance both L2 and cosine can report -- so -/// a search returning fewer neighbours than requested would surface row 0 as its -/// top match. The FFI reports only a status code, never how many slots it wrote, -/// so the Rust side cannot detect a short return; allocating the sentinel is -/// correct either way. -fn new_label_buffer(len: usize) -> Vec { - vec![SENTINEL; len] -} - -/// Post-filter search results to top_k. -fn collect_results( - labels: &[u64], - distances: &[f32], - top_k: usize, - metric: LuminaVectorMetric, -) -> HashMap { - #[derive(PartialEq)] - struct ScoredRow { - row_id: u64, - score: f32, - } - impl Eq for ScoredRow {} - impl PartialOrd for ScoredRow { - fn partial_cmp(&self, other: &Self) -> Option { - Some(self.cmp(other)) - } - } - impl Ord for ScoredRow { - // Reversed on score so the heap top is the weakest candidate; among - // equal scores the larger row id sorts first and is therefore evicted - // first, which keeps the retained set independent of the order the - // searcher returned the pairs in. Same shape as - // `vector_search::ScoredRow`, except that scores are compared through - // `compare_scores` so a NaN cannot claim the strongest slot. - fn cmp(&self, other: &Self) -> std::cmp::Ordering { - compare_scores(other.score, self.score).then_with(|| self.row_id.cmp(&other.row_id)) - } - } - - impl ScoredRow { - fn is_stronger_than(&self, other: &Self) -> bool { - compare_scores(self.score, other.score).then_with(|| other.row_id.cmp(&self.row_id)) - == std::cmp::Ordering::Greater - } - } - - let mut min_heap: BinaryHeap = BinaryHeap::with_capacity(top_k + 1); - for (&row_id, &distance) in labels.iter().zip(distances.iter()) { - if row_id == SENTINEL { - continue; - } - let score = convert_distance_to_score(distance, metric); - let entry = ScoredRow { row_id, score }; - if min_heap.len() < top_k { - min_heap.push(entry); - } else if min_heap - .peek() - .is_some_and(|weakest| entry.is_stronger_than(weakest)) - { - min_heap.pop(); - min_heap.push(entry); - } - } - - let mut result = HashMap::with_capacity(min_heap.len()); - for entry in min_heap { - result.insert(entry.row_id, entry.score); - } - result -} - -pub struct LuminaVectorGlobalIndexReader { - io_meta: GlobalIndexIOMeta, - options: HashMap, - searcher: Option, - index_meta: Option, - search_options: Option>, - max_filter_bytes: Option, - local_index_file: Option, -} - -impl LuminaVectorGlobalIndexReader { - pub fn new(io_meta: GlobalIndexIOMeta, options: HashMap) -> Self { - Self { - io_meta, - options, - searcher: None, - index_meta: None, - search_options: None, - max_filter_bytes: None, - local_index_file: None, - } - } - - pub fn visit_vector_search( - &mut self, - vector_search: &VectorSearch, - stream_fn: impl FnOnce(&str) -> crate::Result, - ) -> crate::Result>> { - self.ensure_loaded(stream_fn)?; - self.search(vector_search) - } - - pub fn visit_batch_vector_search( - &mut self, - vector_searches: &[VectorSearch], - stream_fn: impl FnOnce(&str) -> crate::Result, - ) -> crate::Result>>> { - self.ensure_loaded(stream_fn)?; - self.search_batch(vector_searches) - } - - fn search(&self, vector_search: &VectorSearch) -> crate::Result>> { - let index_meta = self - .index_meta - .as_ref() - .ok_or_else(|| crate::Error::DataInvalid { - message: "index_meta not initialized".to_string(), - source: None, - })?; - let searcher = self - .searcher - .as_ref() - .ok_or_else(|| crate::Error::DataInvalid { - message: "searcher not initialized".to_string(), - source: None, - })?; - let search_options_base = - self.search_options - .as_ref() - .ok_or_else(|| crate::Error::DataInvalid { - message: "search_options not initialized".to_string(), - source: None, - })?; - let max_filter_bytes = self - .max_filter_bytes - .ok_or_else(|| crate::Error::DataInvalid { - message: "max_filter_bytes not initialized".to_string(), - source: None, - })?; - - search_lumina( - searcher, - index_meta, - search_options_base, - vector_search, - max_filter_bytes, - ) - } - - fn search_batch( - &self, - vector_searches: &[VectorSearch], - ) -> crate::Result>>> { - let index_meta = self - .index_meta - .as_ref() - .ok_or_else(|| crate::Error::DataInvalid { - message: "index_meta not initialized".to_string(), - source: None, - })?; - let searcher = self - .searcher - .as_ref() - .ok_or_else(|| crate::Error::DataInvalid { - message: "searcher not initialized".to_string(), - source: None, - })?; - let search_options_base = - self.search_options - .as_ref() - .ok_or_else(|| crate::Error::DataInvalid { - message: "search_options not initialized".to_string(), - source: None, - })?; - let max_filter_bytes = self - .max_filter_bytes - .ok_or_else(|| crate::Error::DataInvalid { - message: "max_filter_bytes not initialized".to_string(), - source: None, - })?; - - search_lumina_batch( - searcher, - index_meta, - search_options_base, - vector_searches, - max_filter_bytes, - ) - } - - fn ensure_loaded( - &mut self, - stream_fn: impl FnOnce(&str) -> crate::Result, - ) -> crate::Result<()> { - if self.searcher.is_some() { - return Ok(()); - } - - let index_meta = LuminaIndexMeta::deserialize(&self.io_meta.metadata)?; - - let max_filter_bytes = max_filter_bytes(&self.options)?; - let searcher_options = merge_searcher_options(&self.options, &index_meta); - - let mut searcher = LuminaSearcher::create(&searcher_options)?; - - let mut stream = stream_fn(&self.io_meta.file_path)?; - let local_index_file = write_temp_index_file(&mut stream)?; - let local_index_path = - local_index_file - .to_str() - .ok_or_else(|| crate::Error::DataInvalid { - message: format!( - "Temporary Lumina index path is not valid UTF-8: {}", - local_index_file.display() - ), - source: None, - })?; - if let Err(err) = searcher.open_file(local_index_path) { - let _ = std::fs::remove_file(&local_index_file); - return Err(err); - } - - self.search_options = Some(searcher_options); - self.max_filter_bytes = Some(max_filter_bytes); - self.index_meta = Some(index_meta); - self.searcher = Some(searcher); - self.local_index_file = Some(local_index_file); - Ok(()) - } - - pub fn close(&mut self) { - self.searcher = None; - self.index_meta = None; - self.search_options = None; - self.max_filter_bytes = None; - if let Some(path) = self.local_index_file.take() { - let _ = std::fs::remove_file(path); - } - } -} - -fn search_lumina( - searcher: &S, - index_meta: &LuminaIndexMeta, - search_options_base: &HashMap, - vector_search: &VectorSearch, - max_filter_bytes: usize, -) -> crate::Result>> { - let expected_dim = index_meta.dim()? as usize; - if vector_search.vector.len() != expected_dim { - return Err(crate::Error::DataInvalid { - message: format!( - "Query vector dimension mismatch: index expects {}, but got {}", - expected_dim, - vector_search.vector.len() - ), - source: None, - }); - } - - let limit = vector_search.limit; - // Java's order: `index.size()` decides first, and the metric is parsed only once - // there is something to search. An empty index has no hits to rank, so reading - // the metric ahead of the size would report a malformed metric for a query whose - // answer is empty whatever the metric says. - let count = searcher.get_count()? as usize; - let effective_k = std::cmp::min(limit, count); - if effective_k == 0 { - return Ok(None); - } - let index_metric = index_meta.metric()?; - - let include_row_ids = vector_search.effective_include_row_ids(); - - let (distances, labels) = if let Some(include_ids) = include_row_ids { - let filter_id_list: Vec = to_scoped_ids(include_ids, max_filter_bytes)?; - if filter_id_list.is_empty() { - return Ok(None); - } - let ek = std::cmp::min(effective_k, filter_id_list.len()); - let mut distances = vec![0.0f32; ek]; - let mut labels = new_label_buffer(ek); - let mut search_opts = search_options_for_query(search_options_base, vector_search); - search_opts.insert("search.thread_safe_filter".to_string(), "true".to_string()); - ensure_search_list_size(&mut search_opts, ek); - searcher.search_with_filter( - &vector_search.vector, - 1, - ek as i32, - &mut distances, - &mut labels, - &filter_id_list, - &search_opts, - )?; - (distances, labels) - } else { - let mut distances = vec![0.0f32; effective_k]; - let mut labels = new_label_buffer(effective_k); - let mut search_opts = search_options_for_query(search_options_base, vector_search); - ensure_search_list_size(&mut search_opts, effective_k); - searcher.search( - &vector_search.vector, - 1, - effective_k as i32, - &mut distances, - &mut labels, - &search_opts, - )?; - (distances, labels) - }; - - let id_to_scores = collect_results(&labels, &distances, effective_k, index_metric); - if id_to_scores.is_empty() { - return Ok(None); - } - - Ok(Some(id_to_scores)) -} - -fn search_lumina_batch( - searcher: &S, - index_meta: &LuminaIndexMeta, - search_options_base: &HashMap, - vector_searches: &[VectorSearch], - max_filter_bytes: usize, -) -> crate::Result>>> { - if vector_searches.is_empty() { - return Ok(Vec::new()); - } - - let limit = vector_searches[0].limit; - let same_limit = vector_searches - .iter() - .all(|vector_search| vector_search.limit == limit); - let shared_filter = same_limit - .then(|| shared_batch_include_row_ids(vector_searches)) - .flatten(); - let has_filter = vector_searches - .iter() - .any(|vector_search| vector_search.effective_include_row_ids().is_some()); - if has_filter && shared_filter.is_none() || !same_limit { - return vector_searches - .iter() - .map(|vector_search| { - search_lumina( - searcher, - index_meta, - search_options_base, - vector_search, - max_filter_bytes, - ) - }) - .collect(); - } - - let expected_dim = index_meta.dim()? as usize; - for vector_search in vector_searches { - if vector_search.vector.len() != expected_dim { - return Err(crate::Error::DataInvalid { - message: format!( - "Query vector dimension mismatch: index expects {}, but got {}", - expected_dim, - vector_search.vector.len() - ), - source: None, - }); - } - } - - // An include-set that permits nothing can match nothing, whatever the index - // holds. Answered from the set itself, so no native call is made at all. - if shared_filter - .as_ref() - .is_some_and(|filter| filter.is_empty()) - { - return Ok(vec![None; vector_searches.len()]); - } - - // Java's order, and it decides what an empty index reports: `index.size()` first, - // then the dense filter is built, and the metric is parsed last. An empty index - // has nothing to return whatever the filter says, so converting the filter first - // would report an oversized one instead, and reading the metric first would - // report a malformed one. Neither is what happened. - let count = searcher.get_count()? as usize; - if count == 0 { - return Ok(vec![None; vector_searches.len()]); - } - let filter_id_list = shared_filter - .map(|include_row_ids| to_scoped_ids(include_row_ids, max_filter_bytes)) - .transpose()?; - let index_metric = index_meta.metric()?; - let effective_k = filter_id_list.as_ref().map_or_else( - || std::cmp::min(limit, count), - |ids| std::cmp::min(std::cmp::min(limit, count), ids.len()), - ); - if effective_k == 0 { - return Ok(vec![None; vector_searches.len()]); - } - - let mut query = Vec::with_capacity(vector_searches.len() * expected_dim); - for vector_search in vector_searches { - query.extend_from_slice(&vector_search.vector); - } - - let mut distances = vec![0.0f32; vector_searches.len() * effective_k]; - let mut labels = new_label_buffer(vector_searches.len() * effective_k); - // `de_vector_read` rejects a batch whose queries disagree on options, so the first - // query's map speaks for the batch, exactly as its `limit` already does above. - let mut search_opts = search_options_for_query(search_options_base, &vector_searches[0]); - ensure_search_list_size(&mut search_opts, effective_k); - if let Some(filter_ids) = filter_id_list { - search_opts.insert("search.thread_safe_filter".to_string(), "true".to_string()); - searcher.search_with_filter( - &query, - vector_searches.len() as i32, - effective_k as i32, - &mut distances, - &mut labels, - &filter_ids, - &search_opts, - )?; - } else { - searcher.search( - &query, - vector_searches.len() as i32, - effective_k as i32, - &mut distances, - &mut labels, - &search_opts, - )?; - } - - let mut results = Vec::with_capacity(vector_searches.len()); - for query_index in 0..vector_searches.len() { - let start = query_index * effective_k; - let end = start + effective_k; - let id_to_scores = collect_results( - &labels[start..end], - &distances[start..end], - effective_k, - index_metric, - ); - if id_to_scores.is_empty() { - results.push(None); - } else { - results.push(Some(id_to_scores)); - } - } - Ok(results) -} - -fn shared_batch_include_row_ids( - vector_searches: &[VectorSearch], -) -> Option<&std::sync::Arc> { - let first = vector_searches.first()?.shared_include_row_ids.as_ref()?; - vector_searches - .iter() - .skip(1) - .all(|vector_search| { - vector_search - .shared_include_row_ids - .as_ref() - .is_some_and(|include_row_ids| std::sync::Arc::ptr_eq(first, include_row_ids)) - }) - .then_some(first) -} - -fn write_temp_index_file(stream: &mut S) -> crate::Result { - stream - .seek(SeekFrom::Start(0)) - .map_err(|e| crate::Error::UnexpectedError { - message: format!("Failed to seek Lumina index stream to start: {}", e), - source: Some(Box::new(e)), - })?; - - let path = std::env::temp_dir().join(format!( - "paimon-lumina-index-{}.index", - uuid::Uuid::new_v4() - )); - let mut file = std::fs::File::create(&path).map_err(|e| crate::Error::UnexpectedError { - message: format!( - "Failed to create temporary Lumina index file '{}': {}", - path.display(), - e - ), - source: Some(Box::new(e)), - })?; - std::io::copy(stream, &mut file).map_err(|e| crate::Error::UnexpectedError { - message: format!( - "Failed to write temporary Lumina index file '{}': {}", - path.display(), - e - ), - source: Some(Box::new(e)), - })?; - file.sync_all().map_err(|e| crate::Error::UnexpectedError { - message: format!( - "Failed to sync temporary Lumina index file '{}': {}", - path.display(), - e - ), - source: Some(Box::new(e)), - })?; - Ok(path) -} - -impl Drop for LuminaVectorGlobalIndexReader { - fn drop(&mut self) { - self.close(); - } -} - -#[cfg(test)] -mod tests { - - #[test] - fn to_scoped_ids_enforces_the_lumina_dense_filter_budget() { - let mut at_limit = roaring::RoaringTreemap::new(); - at_limit.insert_range(0..10u64); - assert_eq!( - to_scoped_ids(&at_limit, 10 * std::mem::size_of::()) - .unwrap() - .len(), - 10 - ); - - let mut over = roaring::RoaringTreemap::new(); - over.insert_range(0..11u64); - let error = to_scoped_ids(&over, 10 * std::mem::size_of::()) - .map(|_| ()) - .expect_err("the dense filter must stay inside its byte budget"); - assert!(error.to_string().contains("max-filter-bytes"), "{error}"); - } - - #[test] - fn to_scoped_ids_keeps_javas_integer_max_value_bound() { - let mut over = roaring::RoaringTreemap::new(); - over.insert_range(0..=(i32::MAX as u64 + 1)); - let error = to_scoped_ids(&over, usize::MAX) - .map(|_| ()) - .expect_err("Java's dense-array index bound must remain in force"); - assert!(error.to_string().contains("exceeds"), "{error}"); - } - - #[test] - fn max_filter_bytes_parses_memory_sizes_and_rejects_bad_values() { - assert_eq!( - max_filter_bytes(&HashMap::new()).unwrap(), - DEFAULT_LUMINA_SEARCH_MAX_FILTER_BYTES - ); - assert_eq!( - max_filter_bytes(&HashMap::from([( - LUMINA_SEARCH_MAX_FILTER_BYTES_OPTION.to_string(), - "2 mb".to_string(), - )])) - .unwrap(), - 2 * 1024 * 1024 - ); - for value in ["0", "invalid"] { - assert!(max_filter_bytes(&HashMap::from([( - LUMINA_SEARCH_MAX_FILTER_BYTES_OPTION.to_string(), - value.to_string(), - )])) - .is_err()); - } - } - - use super::*; - use crate::lumina::{KEY_DIMENSION, KEY_DISTANCE_METRIC}; - use crate::vector_search::GlobalIndexIOMeta; - use std::io::Cursor; - use std::sync::atomic::{AtomicUsize, Ordering}; - use std::sync::{Arc, Mutex}; - - #[derive(Debug, PartialEq)] - struct FilteredSearchCall { - query: Vec, - n: i32, - k: i32, - filter_ids: Vec, - } - - struct RecordingSearcher { - count: u64, - count_calls: AtomicUsize, - unfiltered_calls: Mutex, i32, i32)>>, - filtered_calls: Mutex>, - search_options: Mutex>>, - } - - impl RecordingSearcher { - fn new(count: u64) -> Self { - Self { - count, - count_calls: AtomicUsize::new(0), - unfiltered_calls: Mutex::new(Vec::new()), - filtered_calls: Mutex::new(Vec::new()), - search_options: Mutex::new(Vec::new()), - } - } - } - - impl LuminaSearch for RecordingSearcher { - fn search( - &self, - query: &[f32], - n: i32, - k: i32, - distances: &mut [f32], - labels: &mut [u64], - options: &HashMap, - ) -> crate::Result<()> { - self.search_options - .lock() - .expect("search options lock") - .push(options.clone()); - self.unfiltered_calls - .lock() - .expect("unfiltered call lock") - .push((query.to_vec(), n, k)); - for (index, (distance, label)) in - distances.iter_mut().zip(labels.iter_mut()).enumerate() - { - *distance = index as f32; - *label = index as u64; - } - Ok(()) - } - - fn search_with_filter( - &self, - query: &[f32], - n: i32, - k: i32, - distances: &mut [f32], - labels: &mut [u64], - filter_ids: &[u64], - options: &HashMap, - ) -> crate::Result<()> { - self.search_options - .lock() - .expect("search options lock") - .push(options.clone()); - self.filtered_calls - .lock() - .expect("filtered call lock") - .push(FilteredSearchCall { - query: query.to_vec(), - n, - k, - filter_ids: filter_ids.to_vec(), - }); - for (index, (distance, label)) in - distances.iter_mut().zip(labels.iter_mut()).enumerate() - { - *distance = index as f32; - *label = filter_ids[index % filter_ids.len()]; - } - Ok(()) - } - - fn get_count(&self) -> crate::Result { - self.count_calls.fetch_add(1, Ordering::Relaxed); - Ok(self.count) - } - } - - fn test_index_meta(dim: usize) -> LuminaIndexMeta { - LuminaIndexMeta::new(HashMap::from([ - (KEY_DIMENSION.to_string(), dim.to_string()), - (KEY_DISTANCE_METRIC.to_string(), "l2".to_string()), - ])) - } - - /// Table options that disagree with the built index must not reach the searcher. - /// `validate_and_cap_pq_m` capped `encoding.pq.m` to the column's 32 dimensions - /// when the index was built, so opening it with the table's raw 64 would hand the - /// searcher a PQ geometry the index file does not have. - #[test] - fn test_merge_searcher_options_lets_the_index_win_over_table_options() { - let table_options = HashMap::from([ - ("lumina.encoding.type".to_string(), "pq".to_string()), - ("lumina.encoding.pq.m".to_string(), "64".to_string()), - ("lumina.distance.metric".to_string(), "cosine".to_string()), - ( - "lumina.diskann.search.list_size".to_string(), - "64".to_string(), - ), - ( - LUMINA_SEARCH_MAX_FILTER_BYTES_OPTION.to_string(), - "1 mb".to_string(), - ), - ]); - let index_meta = LuminaIndexMeta::new(HashMap::from([ - (KEY_DIMENSION.to_string(), "32".to_string()), - (KEY_DISTANCE_METRIC.to_string(), "l2".to_string()), - ("encoding.type".to_string(), "pq".to_string()), - ("encoding.pq.m".to_string(), "32".to_string()), - ])); - - let merged = merge_searcher_options(&table_options, &index_meta); - - // The index's own geometry, not the table's. - assert_eq!(merged.get("encoding.pq.m").map(String::as_str), Some("32")); - // Scores are always converted with `index_meta.metric()`, so a table metric - // reaching the searcher would rank by one metric and score by another. - assert_eq!( - merged.get(KEY_DISTANCE_METRIC).map(String::as_str), - Some("l2") - ); - // A key outside `ALL_OPTIONS_DEFAULTS` that this index was not built with - // still comes from the table. - assert_eq!( - merged.get("diskann.search.list_size").map(String::as_str), - Some("64") - ); - // Rust-only reader budget, never a native option. - assert!(!merged.contains_key("search.max-filter-bytes")); - assert!(merged.keys().all(|key| !key.starts_with("lumina."))); - } - - /// Java layers per-query options last (`LuminaVectorGlobalIndexReader:253-255`). - /// Letting the index win over the table must not also let it win over the query, - /// and a Rust-built index records `diskann.search.beam_width` for every index, so - /// the query layer is the only one left that can still tune it. - #[test] - fn test_per_query_options_still_override_the_merged_base() { - let searcher = RecordingSearcher::new(10); - let index_meta = LuminaIndexMeta::new(HashMap::from([ - (KEY_DIMENSION.to_string(), "2".to_string()), - (KEY_DISTANCE_METRIC.to_string(), "l2".to_string()), - ("diskann.search.beam_width".to_string(), "4".to_string()), - ])); - let table_options = HashMap::from([( - "lumina.diskann.search.beam_width".to_string(), - "8".to_string(), - )]); - let base = merge_searcher_options(&table_options, &index_meta); - // The table lost to the index, which is the point of `merge_searcher_options`. - assert_eq!( - base.get("diskann.search.beam_width").map(String::as_str), - Some("4") - ); - - let mut search = VectorSearch::new(vec![1.0, 0.0], 2, "embedding".to_string()).unwrap(); - search.options = HashMap::from([( - "lumina.diskann.search.beam_width".to_string(), - "32".to_string(), - )]); - - search_lumina( - &searcher, - &index_meta, - &base, - &search, - DEFAULT_LUMINA_SEARCH_MAX_FILTER_BYTES, - ) - .expect("search should succeed"); - - let recorded = searcher.search_options.lock().expect("search options lock"); - assert_eq!(recorded.len(), 1); - assert_eq!( - recorded[0] - .get("diskann.search.beam_width") - .map(String::as_str), - Some("32") - ); - } - - #[test] - fn test_shared_filter_uses_one_lumina_batch_search() { - let searcher = RecordingSearcher::new(10); - let shared_filter = Arc::new(roaring::RoaringTreemap::from_iter([2, 4, 6])); - let mut first = VectorSearch::new(vec![1.0, 0.0], 2, "embedding".to_string()).unwrap(); - first.set_shared_include_row_ids(Arc::clone(&shared_filter)); - let mut second = VectorSearch::new(vec![0.0, 1.0], 2, "embedding".to_string()).unwrap(); - second.set_shared_include_row_ids(Arc::clone(&shared_filter)); - - let results = search_lumina_batch( - &searcher, - &test_index_meta(2), - &HashMap::new(), - &[first, second], - DEFAULT_LUMINA_SEARCH_MAX_FILTER_BYTES, - ) - .expect("shared filtered batch search should succeed"); - - assert_eq!(results.len(), 2); - assert!(searcher - .unfiltered_calls - .lock() - .expect("unfiltered call lock") - .is_empty()); - assert_eq!( - *searcher.filtered_calls.lock().expect("filtered call lock"), - vec![FilteredSearchCall { - query: vec![1.0, 0.0, 0.0, 1.0], - n: 2, - k: 2, - filter_ids: vec![2, 4, 6], - }] - ); - } - - #[test] - fn test_equal_but_distinct_filters_keep_scalar_fallback() { - let searcher = RecordingSearcher::new(10); - let mut first = VectorSearch::new(vec![1.0, 0.0], 2, "embedding".to_string()).unwrap(); - first.set_shared_include_row_ids(Arc::new(roaring::RoaringTreemap::from_iter([2, 4, 6]))); - let mut second = VectorSearch::new(vec![0.0, 1.0], 2, "embedding".to_string()).unwrap(); - second.set_shared_include_row_ids(Arc::new(roaring::RoaringTreemap::from_iter([2, 4, 6]))); - - search_lumina_batch( - &searcher, - &test_index_meta(2), - &HashMap::new(), - &[first, second], - DEFAULT_LUMINA_SEARCH_MAX_FILTER_BYTES, - ) - .expect("distinct filtered searches should succeed"); - - let calls = searcher.filtered_calls.lock().expect("filtered call lock"); - assert_eq!(calls.len(), 2); - assert!(calls.iter().all(|call| call.n == 1)); - } - - #[test] - fn test_mixed_filters_and_limits_keep_scalar_fallback() { - let shared_filter = Arc::new(roaring::RoaringTreemap::from_iter([2, 4, 6])); - let mut filtered = VectorSearch::new(vec![1.0, 0.0], 2, "embedding".to_string()).unwrap(); - filtered.set_shared_include_row_ids(Arc::clone(&shared_filter)); - let unfiltered = VectorSearch::new(vec![0.0, 1.0], 2, "embedding".to_string()).unwrap(); - let mixed_searcher = RecordingSearcher::new(10); - - search_lumina_batch( - &mixed_searcher, - &test_index_meta(2), - &HashMap::new(), - &[filtered, unfiltered], - DEFAULT_LUMINA_SEARCH_MAX_FILTER_BYTES, - ) - .expect("mixed filtered searches should succeed"); - - assert_eq!( - mixed_searcher - .filtered_calls - .lock() - .expect("filtered call lock") - .len(), - 1 - ); - assert_eq!( - mixed_searcher - .unfiltered_calls - .lock() - .expect("unfiltered call lock") - .len(), - 1 - ); - - let mut first = VectorSearch::new(vec![1.0, 0.0], 1, "embedding".to_string()).unwrap(); - first.set_shared_include_row_ids(Arc::clone(&shared_filter)); - let mut second = VectorSearch::new(vec![0.0, 1.0], 2, "embedding".to_string()).unwrap(); - second.set_shared_include_row_ids(shared_filter); - let differing_limit_searcher = RecordingSearcher::new(10); - - search_lumina_batch( - &differing_limit_searcher, - &test_index_meta(2), - &HashMap::new(), - &[first, second], - DEFAULT_LUMINA_SEARCH_MAX_FILTER_BYTES, - ) - .expect("differing-limit filtered searches should succeed"); - - let calls = differing_limit_searcher - .filtered_calls - .lock() - .expect("filtered call lock"); - assert_eq!(calls.len(), 2); - assert_eq!(calls.iter().map(|call| call.k).collect::>(), [1, 2]); - assert!(calls.iter().all(|call| call.n == 1)); - } - - #[test] - fn a_pk_vector_live_mask_is_bounded_when_lumina_densifies_it() { - use crate::deletion_vector::DeletionVector; - use crate::spec::PrimaryKeyIndexSourceFile; - use crate::vindex::pkvector::ann::build_live_row_ids; - use roaring::RoaringBitmap; - - let source_files = vec![PrimaryKeyIndexSourceFile::new("file-a".to_string(), 4).unwrap()]; - let active = std::collections::HashSet::from(["file-a".to_string()]); - let mut bitmap = RoaringBitmap::new(); - bitmap.insert(3); - let deletion_vectors = HashMap::from([( - "file-a".to_string(), - Arc::new(DeletionVector::from_bitmap(bitmap)), - )]); - let live = build_live_row_ids(&source_files, &active, &deletion_vectors, None) - .unwrap() - .expect("the deletion vector forces a live-row mask"); - assert_eq!(live.len(), 3); - - let searcher = RecordingSearcher::new(4); - let mut search = VectorSearch::new(vec![1.0, 0.0], 2, "embedding".to_string()).unwrap(); - search.set_shared_include_row_ids(Arc::new(live)); - let error = search_lumina( - &searcher, - &test_index_meta(2), - &HashMap::new(), - &search, - 2 * std::mem::size_of::(), - ) - .expect_err("Lumina must reject the PK mask before calling its native searcher"); - - assert!(error.to_string().contains("max-filter-bytes"), "{error}"); - assert!(searcher - .filtered_calls - .lock() - .expect("filtered call lock") - .is_empty()); - } - - #[test] - fn test_empty_shared_filter_skips_native_search() { - let searcher = RecordingSearcher::new(10); - let shared_filter = Arc::new(roaring::RoaringTreemap::new()); - let mut first = VectorSearch::new(vec![1.0, 0.0], 2, "embedding".to_string()).unwrap(); - first.set_shared_include_row_ids(Arc::clone(&shared_filter)); - let mut second = VectorSearch::new(vec![0.0, 1.0], 2, "embedding".to_string()).unwrap(); - second.set_shared_include_row_ids(Arc::clone(&shared_filter)); - - let results = search_lumina_batch( - &searcher, - &test_index_meta(2), - &HashMap::new(), - &[first, second], - DEFAULT_LUMINA_SEARCH_MAX_FILTER_BYTES, - ) - .expect("empty shared filter should succeed"); - - assert_eq!(results, vec![None, None]); - assert_eq!( - searcher.count_calls.load(Ordering::Relaxed), - 0, - "an empty shared filter should avoid all native searcher calls" - ); - assert!(searcher - .unfiltered_calls - .lock() - .expect("unfiltered call lock") - .is_empty()); - assert!(searcher - .filtered_calls - .lock() - .expect("filtered call lock") - .is_empty()); - } - - /// An index whose metadata names a metric Lumina does not define, so parsing it - /// fails. Everything else about the index is well formed -- only the ORDER the - /// steps run in decides whether the caller sees that failure or an empty answer. - fn test_index_meta_with_unknown_metric(dim: usize) -> LuminaIndexMeta { - LuminaIndexMeta::new(HashMap::from([ - (KEY_DIMENSION.to_string(), dim.to_string()), - (KEY_DISTANCE_METRIC.to_string(), "not-a-metric".to_string()), - ])) - } - - #[test] - fn an_empty_index_reports_empty_before_the_metric_is_parsed() { - // Java's order, scalar path: `index.size()` decides first. A query against an - // index holding nothing has an empty answer whatever the metric says, so - // reporting a malformed metric here would be an error about something that - // could not have changed the result. - let searcher = RecordingSearcher::new(0); - let search = VectorSearch::new(vec![1.0, 0.0], 2, "embedding".to_string()).unwrap(); - let result = search_lumina( - &searcher, - &test_index_meta_with_unknown_metric(2), - &HashMap::new(), - &search, - DEFAULT_LUMINA_SEARCH_MAX_FILTER_BYTES, - ) - .expect("an empty index reports no hits, not a malformed metric"); - assert_eq!(result, None); - assert!(searcher - .unfiltered_calls - .lock() - .expect("unfiltered call lock") - .is_empty()); - } - - /// The two guards meet here: #803 made `index.size()` decide before the filter is - /// built, and this change bounds what building it may allocate. An empty index with - /// an include-set far above the budget must therefore report NO HITS, not an - /// oversized-filter error -- sizing the filter first would turn a query whose answer - /// is empty either way into a failure. Neither guard alone pins this: without the - /// ordering the budget rejects it, and without the budget there is nothing to - /// order against. - #[test] - fn an_empty_index_outranks_a_filter_too_large_to_densify() { - let searcher = RecordingSearcher::new(0); - let mut oversized = roaring::RoaringTreemap::new(); - oversized.insert_range(0..=(i32::MAX as u64 + 1)); - let shared_filter = Arc::new(oversized); - let mut first = VectorSearch::new(vec![1.0, 0.0], 2, "embedding".to_string()).unwrap(); - first.set_shared_include_row_ids(Arc::clone(&shared_filter)); - let mut second = VectorSearch::new(vec![0.0, 1.0], 2, "embedding".to_string()).unwrap(); - second.set_shared_include_row_ids(Arc::clone(&shared_filter)); - - let results = search_lumina_batch( - &searcher, - &test_index_meta(2), - &HashMap::new(), - &[first, second], - DEFAULT_LUMINA_SEARCH_MAX_FILTER_BYTES, - ) - .expect("an empty index reports no hits, not an oversized filter"); - - assert_eq!(results, vec![None, None]); - assert_eq!( - searcher.count_calls.load(Ordering::Relaxed), - 1, - "the index size is what the answer came from" - ); - assert!(searcher - .filtered_calls - .lock() - .expect("filtered call lock") - .is_empty()); - } - - #[test] - fn an_empty_index_reports_empty_before_the_filter_is_densified() { - // Same order on the batch path, where it also keeps the dense filter from - // being built: `index.size()` first, then the filter, then the metric. - let searcher = RecordingSearcher::new(0); - let shared_filter = Arc::new(roaring::RoaringTreemap::from_iter([2, 4, 6])); - let mut first = VectorSearch::new(vec![1.0, 0.0], 2, "embedding".to_string()).unwrap(); - first.set_shared_include_row_ids(Arc::clone(&shared_filter)); - let mut second = VectorSearch::new(vec![0.0, 1.0], 2, "embedding".to_string()).unwrap(); - second.set_shared_include_row_ids(Arc::clone(&shared_filter)); - - let results = search_lumina_batch( - &searcher, - &test_index_meta_with_unknown_metric(2), - &HashMap::new(), - &[first, second], - DEFAULT_LUMINA_SEARCH_MAX_FILTER_BYTES, - ) - .expect("an empty index reports no hits, not a malformed metric"); - - assert_eq!(results, vec![None, None]); - assert_eq!( - searcher.count_calls.load(Ordering::Relaxed), - 1, - "the index size is what the answer came from" - ); - assert!(searcher - .unfiltered_calls - .lock() - .expect("unfiltered call lock") - .is_empty()); - assert!(searcher - .filtered_calls - .lock() - .expect("filtered call lock") - .is_empty()); - } - - #[test] - fn test_convert_distance_to_score() { - assert_eq!(convert_distance_to_score(0.0, LuminaVectorMetric::L2), 1.0); - assert_eq!(convert_distance_to_score(1.0, LuminaVectorMetric::L2), 0.5); - assert_eq!( - convert_distance_to_score(0.0, LuminaVectorMetric::Cosine), - 1.0 - ); - assert_eq!( - convert_distance_to_score(1.0, LuminaVectorMetric::Cosine), - 0.0 - ); - assert_eq!( - convert_distance_to_score(0.75, LuminaVectorMetric::InnerProduct), - 0.75 - ); - } - - #[test] - fn test_ensure_search_list_size() { - let mut opts = HashMap::new(); - ensure_search_list_size(&mut opts, 10); - assert_eq!(opts.get("diskann.search.list_size").unwrap(), "16"); // max(15, 16) - - let mut opts = HashMap::new(); - ensure_search_list_size(&mut opts, 100); - assert_eq!(opts.get("diskann.search.list_size").unwrap(), "150"); // 100*1.5 - - // does not override existing - let mut opts = HashMap::new(); - opts.insert("diskann.search.list_size".to_string(), "999".to_string()); - ensure_search_list_size(&mut opts, 100); - assert_eq!(opts.get("diskann.search.list_size").unwrap(), "999"); - } - - #[test] - fn test_collect_results() { - let labels = vec![0, 1, 2, SENTINEL, 3]; - let distances = vec![0.5, 0.3, 0.1, 0.0, 0.9]; - let result = collect_results(&labels, &distances, 2, LuminaVectorMetric::InnerProduct); - assert_eq!(result.len(), 2); - // top 2 by score: row 3 (0.9) and row 0 (0.5) - assert!(result.contains_key(&3)); - assert!(result.contains_key(&0)); - assert!(!result.contains_key(&2)); // 0.1 is lowest - } - - /// Rows sharing a score must be kept by ascending row id, not by the order - /// the native searcher happened to return them in. Feeding the same tied - /// scores in two different label orders must select the same rows. - #[test] - fn test_collect_results_breaks_ties_by_row_id() { - // Rows 10, 20, 30 all score 0.5; row 40 scores higher and always wins. - let distances = vec![0.9, 0.5, 0.5, 0.5]; - - let forward = collect_results( - &[40, 10, 20, 30], - &distances, - 2, - LuminaVectorMetric::InnerProduct, - ); - let reversed = collect_results( - &[40, 30, 20, 10], - &distances, - 2, - LuminaVectorMetric::InnerProduct, - ); - - let mut forward_ids: Vec = forward.keys().copied().collect(); - forward_ids.sort_unstable(); - let mut reversed_ids: Vec = reversed.keys().copied().collect(); - reversed_ids.sort_unstable(); - - assert_eq!( - forward_ids, reversed_ids, - "tied rows must not depend on label order" - ); - assert_eq!( - forward_ids, - vec![10, 40], - "among equal scores the smallest row id wins" - ); - } - - /// The same invariant when every candidate ties: the retained set is the - /// `top_k` smallest row ids regardless of input order. - #[test] - fn test_collect_results_all_tied_keeps_smallest_row_ids() { - let distances = vec![0.25; 5]; - - let forward = collect_results( - &[1, 2, 3, 4, 5], - &distances, - 3, - LuminaVectorMetric::InnerProduct, - ); - let shuffled = collect_results( - &[4, 1, 5, 3, 2], - &distances, - 3, - LuminaVectorMetric::InnerProduct, - ); - - let mut forward_ids: Vec = forward.keys().copied().collect(); - forward_ids.sort_unstable(); - let mut shuffled_ids: Vec = shuffled.keys().copied().collect(); - shuffled_ids.sort_unstable(); - - assert_eq!(forward_ids, vec![1, 2, 3]); - assert_eq!(shuffled_ids, vec![1, 2, 3]); - } - - /// A NaN score must never outrank a finite one. `f32::total_cmp` alone ranks - /// a positive NaN above every finite value, so using it here would let a NaN - /// score -- which a non-finite stored vector can produce -- take the only - /// top-1 slot. Both NaN signs must lose, in either arrival order. - #[test] - fn test_collect_results_ranks_nan_below_finite_scores() { - for (labels, distances) in [ - (vec![7u64, 8], vec![f32::NAN, 0.5]), - (vec![8u64, 7], vec![0.5, f32::NAN]), - (vec![7u64, 8], vec![-f32::NAN, 0.5]), - (vec![8u64, 7], vec![0.5, -f32::NAN]), - ] { - let result = collect_results(&labels, &distances, 1, LuminaVectorMetric::InnerProduct); - assert_eq!(result.len(), 1); - assert!( - result.contains_key(&8), - "the finite score must win regardless of arrival order, got {result:?}" - ); - } - } - - /// The label buffer handed to the native searcher must start out as - /// [`SENTINEL`], never zero -- row id `0` is a legal result. - #[test] - fn test_new_label_buffer_is_sentinel_filled() { - assert_eq!(new_label_buffer(3), vec![SENTINEL; 3]); - assert!(new_label_buffer(0).is_empty()); - } - - /// A search that returns fewer neighbours than requested leaves the tail of - /// the buffer exactly as it was allocated. Those slots must not surface as - /// rows: with a zero-filled buffer the tail reads as row `0` at distance - /// `0.0`, and `1.0 / (1.0 + 0.0)` is the highest score L2 can produce, so - /// row 0 would be reported as the best match for every short result. - #[test] - fn test_unfilled_label_slots_are_not_reported_as_hits() { - let mut labels = new_label_buffer(4); - let mut distances = vec![0.0f32; 4]; - labels[0] = 11; - distances[0] = 3.0; - labels[1] = 22; - distances[1] = 7.0; - - let result = collect_results(&labels, &distances, 4, LuminaVectorMetric::L2); - - assert_eq!( - result.len(), - 2, - "only the slots the searcher filled may be reported, got {result:?}" - ); - assert!(result.contains_key(&11) && result.contains_key(&22)); - assert!( - !result.contains_key(&0), - "an untouched slot must not become row 0" - ); - } - - #[test] - fn test_reader_new() { - let m = GlobalIndexIOMeta::new("a".into(), 100, vec![]); - let reader = LuminaVectorGlobalIndexReader::new(m, HashMap::new()); - assert!(reader.searcher.is_none()); - } - - #[test] - fn test_write_temp_index_file_copies_stream() { - let bytes = b"lumina-index-bytes".to_vec(); - let mut stream = Cursor::new(bytes.clone()); - stream.seek(SeekFrom::End(0)).unwrap(); - - let path = write_temp_index_file(&mut stream).unwrap(); - let actual = std::fs::read(&path).unwrap(); - std::fs::remove_file(&path).unwrap(); - - assert_eq!(actual, bytes); - } -} diff --git a/crates/paimon/src/table/de_vector_read.rs b/crates/paimon/src/table/de_vector_read.rs index 1704da8ee..cba378d8f 100644 --- a/crates/paimon/src/table/de_vector_read.rs +++ b/crates/paimon/src/table/de_vector_read.rs @@ -18,8 +18,6 @@ //! Reads global vector indexes, scores raw vectors, and materializes global-row-ID results. use crate::io::{FileIO, FileRead}; -use crate::lumina::reader::LuminaVectorGlobalIndexReader; -use crate::lumina::{LuminaIndexMeta, LuminaVectorMetric}; use crate::spec::{ row_id_data_field, CoreOptions, DataField, DataType, FileKind, GlobalIndexSearchMode, IndexFileMeta, IndexManifestEntry, Predicate, ROW_ID_FIELD_NAME, @@ -41,7 +39,7 @@ use crate::table::vector_read::Read; use crate::table::vector_search_common::{ configured_refine_factor, configured_vector_index_type, indexed_search_limit, log_vindex_range_io_stats, normalize_metric, resolve_materialize_read_type, - validate_global_vector_index_types, vindex_concurrency_limits, VectorIndexBackend, + validate_global_vector_index_types, vindex_concurrency_limits, }; use crate::table::{ find_field_id_by_name, merge_row_ranges, ArrowRecordBatchStream, RowRange, Table, @@ -299,7 +297,7 @@ async fn execute_vector_index_work( evaluation: VectorSearchEvaluation<'_>, work: VectorIndexWork, metric_check: VectorMetricCheck, - range_read_limiter: Option, + range_read_limiter: RangeReadLimiter, batch_index_parallelism: usize, ) -> crate::Result<(Vec, IndexSearchTiming)> { let timing_enabled = vector_search_timing_enabled(); @@ -312,8 +310,11 @@ async fn execute_vector_index_work( .ok_or_else(|| { crate::table::search_task::invalid("Vector task is missing index metadata") })?; - let backend = VectorIndexBackend::from_index_type(&work.entry.index_file.index_type) - .ok_or_else(|| crate::table::search_task::invalid("Unsupported vector task index"))?; + if !is_vindex_index_type(&work.entry.index_file.index_type) { + return Err(crate::table::search_task::invalid( + "Unsupported vector task index", + )); + } let table_path = evaluation.table_path.trim_end_matches('/'); let path = IndexFileLocation::Global { table_path }.resolve( &work.entry.index_file.file_name, @@ -345,110 +346,81 @@ async fn execute_vector_index_work( let mut file_reader_open = Duration::ZERO; let mut full_file_read = None; let io_meta = GlobalIndexIOMeta::new(file_name.clone(), file_size, index_meta_bytes); - let results = match backend { - VectorIndexBackend::Lumina => { + let results = match tokio::runtime::Handle::try_current() { + Ok(runtime) => { + let file_reader_open_start = timing_enabled.then(Instant::now); + let file_reader = input + .reader() + .await + .map_err(|e| crate::Error::DataInvalid { + message: format!( + "Failed to open vindex file '{}' for range reads: {}", + file_name, e + ), + source: None, + })?; + file_reader_open = + file_reader_open_start.map_or(Duration::ZERO, |start| start.elapsed()); + let source = VindexFileReader::new_with_limiter( + Arc::new(file_reader), + runtime, + range_read_limiter, + file_size, + file_name.clone(), + ); + let range_io_stats = source.range_io_stats(); + let results = execute_vindex_searches( + io_meta, + options, + vector_searches, + source, + file_name.clone(), + VectorIndexSearchContext { + parallelism: batch_index_parallelism, + metrics: metric_check, + }, + permit, + ) + .await?; + if let Some(stats) = range_io_stats { + log_vindex_range_io_stats(&file_name, query_count, &stats); + } + results + } + Err(_) if query_count > 1 => { let read_start = timing_enabled.then(Instant::now); let data = input.read().await.map_err(|e| crate::Error::DataInvalid { - message: format!( - "Failed to read {} index file '{}': {}", - backend.error_name(), - file_name, - e - ), + message: format!("Failed to read vindex index file '{}': {}", file_name, e), source: None, })?; if let Some(start) = read_start { full_file_read = Some((start.elapsed(), data.len())); } - let metric = LuminaIndexMeta::deserialize(&io_meta.metadata)?.metric()?; - metric_check.check(RawVectorMetric::from_lumina(metric))?; - execute_global_index_with_guard( - "Lumina global-index batch search task failed", - permit, - move || { - let mut reader = LuminaVectorGlobalIndexReader::new(io_meta, options); - reader.visit_batch_vector_search(&vector_searches, |_| Ok(Cursor::new(data))) + execute_vindex_searches( + io_meta, + options, + vector_searches, + Cursor::new(data), + file_name.clone(), + VectorIndexSearchContext { + parallelism: batch_index_parallelism, + metrics: metric_check, }, + permit, ) .await? } - VectorIndexBackend::Vindex => match tokio::runtime::Handle::try_current() { - Ok(runtime) => { - let file_reader_open_start = timing_enabled.then(Instant::now); - let file_reader = input - .reader() - .await - .map_err(|e| crate::Error::DataInvalid { - message: format!( - "Failed to open vindex file '{}' for range reads: {}", - file_name, e - ), - source: None, - })?; - file_reader_open = - file_reader_open_start.map_or(Duration::ZERO, |start| start.elapsed()); - let source = VindexFileReader::new_with_limiter( - Arc::new(file_reader), - runtime, - range_read_limiter.expect("Vindex range-read limiter"), - file_size, - file_name.clone(), - ); - let range_io_stats = source.range_io_stats(); - let results = execute_vindex_searches( - io_meta, - options, - vector_searches, - source, - file_name.clone(), - VectorIndexSearchContext { - parallelism: batch_index_parallelism, - metrics: metric_check, - }, - permit, - ) - .await?; - if let Some(stats) = range_io_stats { - log_vindex_range_io_stats(&file_name, query_count, &stats); - } - results - } - Err(_) if query_count > 1 => { - let read_start = timing_enabled.then(Instant::now); - let data = input.read().await.map_err(|e| crate::Error::DataInvalid { - message: format!("Failed to read vindex index file '{}': {}", file_name, e), - source: None, - })?; - if let Some(start) = read_start { - full_file_read = Some((start.elapsed(), data.len())); - } - execute_vindex_searches( - io_meta, - options, - vector_searches, - Cursor::new(data), - file_name.clone(), - VectorIndexSearchContext { - parallelism: batch_index_parallelism, - metrics: metric_check, - }, - permit, - ) - .await? - } - Err(error) => { - return Err(crate::Error::UnexpectedError { - message: "Vector index range reader requires a Tokio runtime".to_string(), - source: Some(Box::new(error)), - }); - } - }, + Err(error) => { + return Err(crate::Error::UnexpectedError { + message: "Vector index range reader requires a Tokio runtime".to_string(), + source: Some(Box::new(error)), + }); + } }; if let Some((read, returned_bytes)) = full_file_read { log::debug!( target: "paimon::vector_search", - "event=paimon_vector_full_file_io backend={} file={} nq={} requested_bytes={} returned_bytes={} read_ms={:.3}", - backend.error_name(), + "event=paimon_vector_full_file_io backend=vindex file={} nq={} requested_bytes={} returned_bytes={} read_ms={:.3}", file_name, query_count, file_size, @@ -744,7 +716,7 @@ async fn evaluate_batch_vector_search_with_executor( .iter() .filter(|e| { e.kind == FileKind::Add - && VectorIndexBackend::from_index_type(&e.index_file.index_type).is_some() + && is_vindex_index_type(&e.index_file.index_type) && e.index_file .global_index_meta .as_ref() @@ -957,20 +929,9 @@ async fn evaluate_batch_vector_search_with_executor( }); } ensure_global_index_executor_capacity(concurrency); - let vindex_entry_count = vector_entries - .iter() - .filter(|entry| is_vindex_index_type(&entry.index_file.index_type)) - .count(); - let (batch_index_parallelism, range_read_limiter) = if vindex_entry_count == 0 { - (1, None) - } else { - let (index_parallelism, range_read_concurrency) = - vindex_concurrency_limits(&core_options, vindex_entry_count, concurrency)?; - ( - index_parallelism, - Some(RangeReadLimiter::new(range_read_concurrency)), - ) - }; + let (batch_index_parallelism, range_read_concurrency) = + vindex_concurrency_limits(&core_options, vector_entries.len(), concurrency)?; + let range_read_limiter = RangeReadLimiter::new(range_read_concurrency); let mut works = vector_search_plans .into_iter() .map(|(entry, shared_local_filter)| { @@ -1307,7 +1268,7 @@ pub(super) fn scalar_unindexed_ranges( } fn is_vector_global_index_file(index_file: &IndexFileMeta) -> bool { - VectorIndexBackend::from_index_type(&index_file.index_type).is_some() + is_vindex_index_type(&index_file.index_type) } /// Collect materialized DE rows, join each row's `(rank, score)` by its global @@ -1694,14 +1655,6 @@ impl RawVectorMetric { } } - fn from_lumina(metric: LuminaVectorMetric) -> Self { - match metric { - LuminaVectorMetric::L2 => Self::L2, - LuminaVectorMetric::Cosine => Self::Cosine, - LuminaVectorMetric::InnerProduct => Self::InnerProduct, - } - } - fn from_vindex(metric: MetricType) -> Self { match metric { MetricType::L2 => Self::L2, @@ -1725,7 +1678,7 @@ async fn resolve_raw_vector_metric( .iter() .filter(|entry| entry.kind == FileKind::Add) .find(|entry| { - VectorIndexBackend::from_index_type(&entry.index_file.index_type).is_some() + is_vindex_index_type(&entry.index_file.index_type) && entry .index_file .global_index_meta @@ -1765,14 +1718,12 @@ async fn resolve_raw_vector_metric( return Ok(metric); } // Before files exist, Java creates the configured indexer for raw search. - // NativeVectorGlobalIndexer and LuminaVectorIndexOptions both default to IP. - Ok( - if index_type.is_some_and(|kind| VectorIndexBackend::from_index_type(kind).is_some()) { - RawVectorMetric::InnerProduct - } else { - RawVectorMetric::L2 - }, - ) + // The vindex option resolver defaults to inner product. + Ok(if index_type.is_some_and(is_vindex_index_type) { + RawVectorMetric::InnerProduct + } else { + RawVectorMetric::L2 + }) } /// Rerouted raw searches still validate each shard's persisted metric. @@ -1784,66 +1735,50 @@ async fn read_index_vector_metric( let Some(global_meta) = entry.index_file.global_index_meta.as_ref() else { return Ok(None); }; - let Some(backend) = VectorIndexBackend::from_index_type(&entry.index_file.index_type) else { + if !is_vindex_index_type(&entry.index_file.index_type) { return Ok(None); - }; - match backend { - VectorIndexBackend::Lumina => { - if let Some(index_meta) = global_meta.index_meta.as_ref() { - if !index_meta.is_empty() { - let metric = LuminaIndexMeta::deserialize(index_meta)?.metric()?; - return Ok(Some(RawVectorMetric::from_lumina(metric))); + } + if let Some(index_meta) = global_meta.index_meta.as_ref() { + if let Ok(options) = serde_json::from_slice::>(index_meta) { + if let Some(metric) = options.get("metric") { + if let Some(metric) = RawVectorMetric::parse_normalized(&normalize_metric(metric)) { + return Ok(Some(metric)); } } } - VectorIndexBackend::Vindex => { - if let Some(index_meta) = global_meta.index_meta.as_ref() { - if let Ok(options) = serde_json::from_slice::>(index_meta) { - if let Some(metric) = options.get("metric") { - if let Some(metric) = - RawVectorMetric::parse_normalized(&normalize_metric(metric)) - { - return Ok(Some(metric)); - } - } - } - } - let path = IndexFileLocation::Global { table_path }.resolve( - &entry.index_file.file_name, - entry.index_file.external_path.as_deref(), - ); - let input = file_io.new_input(&path)?; - let read_error = |e| crate::Error::DataInvalid { - message: format!( - "Failed to read vindex index file '{}' for raw search metric: {}", - entry.index_file.file_name, e - ), - source: Some(Box::new(e)), - }; - let header_size = if entry.index_file.file_size > 0 { - (entry.index_file.file_size as u64).min(DISKANN_HEADER_SIZE as u64) - } else { - input - .metadata() - .await - .map_err(&read_error)? - .size - .min(DISKANN_HEADER_SIZE as u64) - }; - let file_reader = input.reader().await.map_err(&read_error)?; - let bytes = file_reader.read(0..header_size).await.map_err(read_error)?; - let reader = - VIndexReader::open(Cursor::new(bytes)).map_err(|e| crate::Error::DataInvalid { - message: format!( - "Failed to open paimon-vindex-core reader for raw search metric: {}", - e - ), - source: Some(Box::new(e)), - })?; - return Ok(Some(RawVectorMetric::from_vindex(reader.metadata().metric))); - } } - Ok(None) + let path = IndexFileLocation::Global { table_path }.resolve( + &entry.index_file.file_name, + entry.index_file.external_path.as_deref(), + ); + let input = file_io.new_input(&path)?; + let read_error = |e| crate::Error::DataInvalid { + message: format!( + "Failed to read vindex index file '{}' for raw search metric: {}", + entry.index_file.file_name, e + ), + source: Some(Box::new(e)), + }; + let header_size = if entry.index_file.file_size > 0 { + (entry.index_file.file_size as u64).min(DISKANN_HEADER_SIZE as u64) + } else { + input + .metadata() + .await + .map_err(&read_error)? + .size + .min(DISKANN_HEADER_SIZE as u64) + }; + let file_reader = input.reader().await.map_err(&read_error)?; + let bytes = file_reader.read(0..header_size).await.map_err(read_error)?; + let reader = VIndexReader::open(Cursor::new(bytes)).map_err(|e| crate::Error::DataInvalid { + message: format!( + "Failed to open paimon-vindex-core reader for raw search metric: {}", + e + ), + source: Some(Box::new(e)), + })?; + Ok(Some(RawVectorMetric::from_vindex(reader.metadata().metric))) } fn vector_metric_override( @@ -1902,7 +1837,6 @@ fn direct_raw_vector_metric_override( } direct_keys.extend([ "test.vector.metric".to_string(), - "lumina.distance.metric".to_string(), "distance.metric".to_string(), "metric".to_string(), ]); @@ -2899,7 +2833,7 @@ pub(super) async fn execute_search_task( queries: work.queries.restore()?, }, metric_check.clone(), - Some(RangeReadLimiter::new(range_concurrency)), + RangeReadLimiter::new(range_concurrency), parallelism, ) .await?; diff --git a/crates/paimon/src/table/de_vector_read/tests.rs b/crates/paimon/src/table/de_vector_read/tests.rs index 218b660b2..c3e80c85f 100644 --- a/crates/paimon/src/table/de_vector_read/tests.rs +++ b/crates/paimon/src/table/de_vector_read/tests.rs @@ -17,13 +17,11 @@ use super::*; use crate::io::{FileIO, FileIOBuilder}; -use crate::lumina::{LEGACY_LUMINA_VECTOR_ANN_IDENTIFIER, LUMINA_IDENTIFIER}; use crate::spec::{ BinaryRow, CoreOptions, DataField, DataType, FileKind, GlobalIndexMeta, IndexFileMeta, IndexManifestEntry, IntType, ROW_ID_FIELD_NAME, }; use crate::table::pk_vector_position_read::{PKEY_VECTOR_POSITION_COLUMN, SEARCH_SCORE_COLUMN}; -use crate::table::vector_search_common::VectorIndexBackend; use crate::table::vector_search_common::{collect_ranked_rows, reorder_and_strip_position}; use crate::table::vector_search_test_utils::{ build_vindex_segment_bytes, de_vector_table, id_gt_filter, pk_vector_table, @@ -65,7 +63,7 @@ fn eval_context<'a>( } } -fn make_lumina_entry( +fn make_vector_entry( file_name: &str, index_type: &str, kind: FileKind, @@ -482,8 +480,6 @@ async fn raw_search_without_files_uses_the_configured_indexer_default() { for (index_type, expected) in [ (" IVF-FLAT ", RawVectorMetric::InnerProduct), ("diskann", RawVectorMetric::InnerProduct), - ("lumina", RawVectorMetric::InnerProduct), - ("lumina-vector-ann", RawVectorMetric::InnerProduct), ] { let options = HashMap::from([( "fields.embedding.index-type".to_string(), @@ -560,8 +556,8 @@ async fn mixed_vector_formats_fail_before_opening_files() { let file_io = FileIOBuilder::new("memory").build().unwrap(); let fields = vec![make_field(2, "embedding")]; let entries = [ - make_lumina_entry("missing-flat", IVF_FLAT_IDENTIFIER, FileKind::Add, 2), - make_lumina_entry("missing-diskann", "diskann", FileKind::Add, 2), + make_vector_entry("missing-flat", IVF_FLAT_IDENTIFIER, FileKind::Add, 2), + make_vector_entry("missing-diskann", "diskann", FileKind::Add, 2), ]; let query = VectorSearch::new(vec![1.0, 0.0], 1, "embedding".to_string()).unwrap(); let error = evaluate_vector_search( @@ -590,7 +586,7 @@ async fn indexed_vector_metric_ignores_unrelated_column_options() { .write(bytes::Bytes::from(index.clone())) .await .unwrap(); - let mut entry = make_lumina_entry("test.idx", IVF_FLAT_IDENTIFIER, FileKind::Add, 2); + let mut entry = make_vector_entry("test.idx", IVF_FLAT_IDENTIFIER, FileKind::Add, 2); entry.index_file.file_size = index.len() as i64; let fields = vec![make_field(2, "embedding")]; let options = HashMap::from([("fields.other.metric".to_string(), "l2".to_string())]); @@ -624,7 +620,7 @@ async fn indexed_vector_metric_ignores_unrelated_column_options() { #[tokio::test] async fn test_resolve_raw_vector_metric_uses_vindex_manifest_metadata() { let file_io = FileIOBuilder::new("memory").build().unwrap(); - let mut entry = make_lumina_entry("missing.idx", IVF_FLAT_IDENTIFIER, FileKind::Add, 2); + let mut entry = make_vector_entry("missing.idx", IVF_FLAT_IDENTIFIER, FileKind::Add, 2); let index_meta = serde_json::to_vec(&HashMap::from([( "metric".to_string(), "cosine".to_string(), @@ -667,7 +663,7 @@ async fn test_resolve_raw_vector_metric_falls_back_to_vindex_header() { (0, b"{}".to_vec()), (-1, b"{}".to_vec()), ] { - let mut entry = make_lumina_entry("test.idx", IVF_FLAT_IDENTIFIER, FileKind::Add, 2); + let mut entry = make_vector_entry("test.idx", IVF_FLAT_IDENTIFIER, FileKind::Add, 2); entry.index_file.file_size = file_size; entry .index_file @@ -730,7 +726,7 @@ fn test_configured_refine_factor_precedence_and_aliases() { &HashMap::new(), &global_options, "embedding", - LUMINA_IDENTIFIER, + crate::vindex::DISKANN_IDENTIFIER, ) .unwrap(), 4 @@ -744,7 +740,7 @@ fn test_configured_refine_factor_rejects_invalid_values() { &zero_options, &HashMap::new(), "embedding", - LUMINA_IDENTIFIER, + crate::vindex::DISKANN_IDENTIFIER, ) .unwrap_err(); assert!(err.to_string().contains("must be positive")); @@ -754,7 +750,7 @@ fn test_configured_refine_factor_rejects_invalid_values() { &invalid_options, &HashMap::new(), "embedding", - LUMINA_IDENTIFIER, + crate::vindex::DISKANN_IDENTIFIER, ) .unwrap_err(); assert!(err.to_string().contains("Must be an integer")); @@ -934,12 +930,7 @@ async fn test_batch_evaluate_no_matching_field_returns_empty_per_query() { ]; let options = HashMap::new(); - let entry = make_lumina_entry( - "test.idx", - LEGACY_LUMINA_VECTOR_ANN_IDENTIFIER, - FileKind::Add, - 99, - ); + let entry = make_vector_entry("test.idx", IVF_FLAT_IDENTIFIER, FileKind::Add, 99); let results = evaluate_batch_vector_search( eval_context(&file_io, &options, &fields, None), @@ -993,7 +984,7 @@ async fn test_evaluate_ignores_non_vector_index_type() { let vs = VectorSearch::new(vec![1.0], 10, "embedding".to_string()).unwrap(); let options = HashMap::new(); - let entry = make_lumina_entry("test.idx", "btree", FileKind::Add, 2); + let entry = make_vector_entry("test.idx", "btree", FileKind::Add, 2); let result = evaluate_vector_search( eval_context(&file_io, &options, &fields, None), @@ -1033,12 +1024,7 @@ async fn test_evaluate_no_matching_field() { let vs = VectorSearch::new(vec![1.0], 10, "embedding".to_string()).unwrap(); let options = HashMap::new(); - let entry = make_lumina_entry( - "test.idx", - LEGACY_LUMINA_VECTOR_ANN_IDENTIFIER, - FileKind::Add, - 99, - ); + let entry = make_vector_entry("test.idx", IVF_FLAT_IDENTIFIER, FileKind::Add, 99); let result = evaluate_vector_search( eval_context(&file_io, &options, &fields, None), @@ -1057,12 +1043,7 @@ async fn test_evaluate_skips_delete_entries() { let vs = VectorSearch::new(vec![1.0], 10, "embedding".to_string()).unwrap(); let options = HashMap::new(); - let entry = make_lumina_entry( - "test.idx", - LEGACY_LUMINA_VECTOR_ANN_IDENTIFIER, - FileKind::Delete, - 2, - ); + let entry = make_vector_entry("test.idx", IVF_FLAT_IDENTIFIER, FileKind::Delete, 2); let result = evaluate_vector_search( eval_context(&file_io, &options, &fields, None), @@ -1074,57 +1055,6 @@ async fn test_evaluate_skips_delete_entries() { assert!(result.is_empty()); } -#[tokio::test] -async fn test_evaluate_accepts_canonical_lumina_index_type() { - let file_io = crate::io::FileIOBuilder::new("memory").build().unwrap(); - let fields = vec![make_field(2, "embedding")]; - let vs = VectorSearch::new(vec![1.0], 10, "embedding".to_string()).unwrap(); - let options = HashMap::new(); - - let entry = make_lumina_entry("missing.idx", LUMINA_IDENTIFIER, FileKind::Add, 2); - - let err = evaluate_vector_search( - eval_context(&file_io, &options, &fields, None), - &[entry], - &vs, - ) - .await - .unwrap_err(); - assert!( - err.to_string() - .contains("Failed to read Lumina index file 'missing.idx'"), - "unexpected error: {err}" - ); -} - -#[tokio::test] -async fn test_evaluate_accepts_legacy_lumina_index_type() { - let file_io = crate::io::FileIOBuilder::new("memory").build().unwrap(); - let fields = vec![make_field(2, "embedding")]; - let vs = VectorSearch::new(vec![1.0], 10, "embedding".to_string()).unwrap(); - let options = HashMap::new(); - - let entry = make_lumina_entry( - "missing.idx", - LEGACY_LUMINA_VECTOR_ANN_IDENTIFIER, - FileKind::Add, - 2, - ); - - let err = evaluate_vector_search( - eval_context(&file_io, &options, &fields, None), - &[entry], - &vs, - ) - .await - .unwrap_err(); - assert!( - err.to_string() - .contains("Failed to read Lumina index file 'missing.idx'"), - "unexpected error: {err}" - ); -} - #[tokio::test] async fn test_evaluate_accepts_vindex_index_type() { let file_io = crate::io::FileIOBuilder::new("memory").build().unwrap(); @@ -1132,7 +1062,7 @@ async fn test_evaluate_accepts_vindex_index_type() { let vs = VectorSearch::new(vec![1.0], 10, "embedding".to_string()).unwrap(); let options = HashMap::new(); - let entry = make_lumina_entry("missing.idx", IVF_FLAT_IDENTIFIER, FileKind::Add, 2); + let entry = make_vector_entry("missing.idx", IVF_FLAT_IDENTIFIER, FileKind::Add, 2); let err = evaluate_vector_search( eval_context(&file_io, &options, &fields, None), @@ -1165,7 +1095,7 @@ fn test_single_vindex_outside_tokio_returns_error() { let fields = vec![make_field(2, "embedding")]; let vs = VectorSearch::new(vec![1.0], 10, "embedding".to_string()).unwrap(); let options = HashMap::new(); - let entry = make_lumina_entry("test.idx", IVF_FLAT_IDENTIFIER, FileKind::Add, 2); + let entry = make_vector_entry("test.idx", IVF_FLAT_IDENTIFIER, FileKind::Add, 2); let err = evaluate_vector_search( eval_context(&file_io, &options, &fields, None), @@ -1200,7 +1130,7 @@ fn test_batch_vindex_outside_tokio_uses_buffered_fallback() { VectorSearch::new(vec![0.0, 1.0], 2, "embedding".to_string()).unwrap(), ]; let options = HashMap::new(); - let mut entry = make_lumina_entry("test.idx", IVF_FLAT_IDENTIFIER, FileKind::Add, 2); + let mut entry = make_vector_entry("test.idx", IVF_FLAT_IDENTIFIER, FileKind::Add, 2); entry.index_file.file_size = index.len() as i64; entry.index_file.row_count = 3; entry @@ -1223,28 +1153,6 @@ fn test_batch_vindex_outside_tokio_uses_buffered_fallback() { }); } -#[test] -fn from_index_type_classifies_lumina_and_vindex() { - assert_eq!( - VectorIndexBackend::from_index_type("lumina"), - Some(VectorIndexBackend::Lumina) - ); - assert_eq!( - VectorIndexBackend::from_index_type("lumina-vector-ann"), - Some(VectorIndexBackend::Lumina) - ); - assert_eq!( - VectorIndexBackend::from_index_type("ivf-flat"), - Some(VectorIndexBackend::Vindex) - ); - for index_type in ["ivf-sq", "ivf-rq", "diskann"] { - assert_eq!( - VectorIndexBackend::from_index_type(index_type), - Some(VectorIndexBackend::Vindex) - ); - } -} - #[tokio::test] async fn execute_filter_on_empty_de_path_returns_empty() { // No PK-vector index and no snapshot: the request follows the @@ -1615,7 +1523,7 @@ async fn indexed_search_rejects_mixed_metrics_and_conflicting_query_metric() { for (ordinal, metric) in ["l2", "cosine"].into_iter().enumerate() { let bytes = build_vindex_segment_bytes(metric); let name = format!("metric-{ordinal}.idx"); - let mut entry = make_lumina_entry(&name, IVF_FLAT_IDENTIFIER, FileKind::Add, 2); + let mut entry = make_vector_entry(&name, IVF_FLAT_IDENTIFIER, FileKind::Add, 2); entry.index_file.file_size = bytes.len() as i64; let meta = entry.index_file.global_index_meta.as_mut().unwrap(); meta.row_range_start = ordinal as i64 * 3; @@ -1750,8 +1658,8 @@ fn scalar_unindexed_coverage_intersects_all_predicate_fields() { let options = HashMap::new(); let fields = vec![make_field(1, "id"), make_field(2, "other")]; let evaluation = eval_context(&io, &options, &fields, Some(20)); - let mut id = make_lumina_entry("id.btree", "btree", FileKind::Add, 1); - let mut other = make_lumina_entry("other.btree", "btree", FileKind::Add, 2); + let mut id = make_vector_entry("id.btree", "btree", FileKind::Add, 1); + let mut other = make_vector_entry("other.btree", "btree", FileKind::Add, 2); other .index_file .global_index_meta diff --git a/crates/paimon/src/table/global_index_build_builder.rs b/crates/paimon/src/table/global_index_build_builder.rs index ab46595c5..2f31ad69c 100644 --- a/crates/paimon/src/table/global_index_build_builder.rs +++ b/crates/paimon/src/table/global_index_build_builder.rs @@ -26,14 +26,12 @@ use super::global_index_types::{ normalize_global_index_type, normalize_queryable_global_index_type, BTREE_GLOBAL_INDEX_TYPE, FULL_TEXT_GLOBAL_INDEX_TYPE, }; -use super::lumina_index_build_builder::LuminaIndexBuildBuilder; use super::sorted_global_index_build_builder::SortedGlobalIndexBuildBuilder; use super::vindex_index_build_builder::timing::{ vector_index_build_timing_enabled, VectorIndexBuildTiming, }; use super::vindex_index_build_builder::VindexIndexBuildBuilder; use super::{CommitMessage, Table, TableCommit}; -use crate::lumina::LUMINA_IDENTIFIER; use crate::spec::{CoreOptions, Predicate}; use crate::{Error, Result}; @@ -241,14 +239,6 @@ impl<'a> GlobalIndexBuildBuilder<'a> { message: "Full-text global index build requires the 'fulltext' feature".into(), }); } - if index_type == LUMINA_IDENTIFIER { - let mut builder = LuminaIndexBuildBuilder::new(self.table); - builder - .with_index_column(columns[0]) - .with_options(self.options.clone()); - builder.partition_filter = self.partition_filter.clone(); - return Ok(builder.prepare().await?.into()); - } let mut builder = VindexIndexBuildBuilder::new(self.table, index_type); builder .with_index_column(columns[0]) @@ -303,8 +293,6 @@ mod tests { ("multivalue", "vector"), ("fm", "text"), ("full-text", "text"), - ("lumina", "vector"), - ("lumina-vector-ann", "vector"), ("ivf-flat", "vector"), ("ivf-pq", "vector"), ("ivf-sq", "vector"), @@ -314,17 +302,7 @@ mod tests { if !GlobalIndexBuildBuilder::supports_index_type(kind) { continue; } - let mut options = HashMap::from([( - format!( - "{}.dimension", - if kind == "lumina-vector-ann" { - "lumina" - } else { - kind - } - ), - "2".into(), - )]); + let mut options = HashMap::from([(format!("{kind}.dimension"), "2".into())]); if kind == "ivf-pq" { options.insert("ivf-pq.pq.m".into(), "1".into()); } @@ -432,7 +410,7 @@ mod tests { #[tokio::test] async fn column_validation_precedes_any_index_io() { let table = empty_table(); - for kind in ["bitmap", "fm", "full-text", "ivf-flat", "lumina"] { + for kind in ["bitmap", "fm", "full-text", "ivf-flat", "diskann"] { let mut builder = table.new_global_index_build_builder(); builder .with_index_columns(&["id", "text"]) diff --git a/crates/paimon/src/table/global_index_build_common.rs b/crates/paimon/src/table/global_index_build_common.rs index be66da97b..eba97e802 100644 --- a/crates/paimon/src/table/global_index_build_common.rs +++ b/crates/paimon/src/table/global_index_build_common.rs @@ -16,7 +16,7 @@ // under the License. //! Shared helpers for the global-index build builders (btree, full-text, -//! lumina, vindex). +//! vindex). //! //! Mirrors Java `GlobalIndexBuilderUtils`, which exposes a single //! `indexedRowRanges` used by both the sorted and vector index builders. The @@ -25,17 +25,22 @@ //! `index_type` string, so it lives here once rather than being copied into //! each builder. +#[cfg(feature = "fulltext")] use crate::io::OutputFile; use crate::spec::{DataField, FileKind, IndexManifest}; use crate::table::{merge_row_ranges, RowRange, Table}; use crate::{Error, Result}; +#[cfg(feature = "fulltext")] use bytes::Bytes; +#[cfg(feature = "fulltext")] use std::path::Path; +#[cfg(feature = "fulltext")] use tokio::io::AsyncReadExt; pub(crate) mod preparation; pub(crate) mod vector; +#[cfg(feature = "fulltext")] const COPY_BUFFER_SIZE: usize = 1024 * 1024; /// Validate the table that owns global row IDs. Procedure options only tune @@ -65,9 +70,9 @@ pub(crate) fn validate_build_table(table: &Table) -> Result<()> { Ok(()) } -/// Stream a locally built index file to its table location. Native index -/// builders (Lumina, full-text) serialize to a local temporary file first -/// because their writers need synchronous I/O. +/// Stream a locally built full-text index to its table location. The native +/// writer serializes to a temporary file because it needs synchronous I/O. +#[cfg(feature = "fulltext")] pub(crate) async fn copy_local_file_to_output( source_path: &Path, output: OutputFile, diff --git a/crates/paimon/src/table/global_index_drop_builder.rs b/crates/paimon/src/table/global_index_drop_builder.rs index ddd7ca4e6..9aca59ff6 100644 --- a/crates/paimon/src/table/global_index_drop_builder.rs +++ b/crates/paimon/src/table/global_index_drop_builder.rs @@ -460,11 +460,11 @@ mod tests { refused( table .new_global_index_build_builder() - .with_index_type("lumina") + .with_index_type("diskann") .execute() .await .unwrap_err(), - "building a lumina index", + "building a diskann index", ); } @@ -942,7 +942,6 @@ mod tests { "multivalue", "fm", "full-text", - "lumina", "ivf-flat", "ivf-pq", "ivf-sq", @@ -1192,44 +1191,6 @@ mod tests { assert_eq!(remaining[0].index_file.file_name, "btree-name.index"); } - #[tokio::test] - async fn test_drop_lumina_matches_legacy_alias_entry() { - let table = test_table("memory:/test_drop_lumina_legacy"); - setup_dirs(&table).await; - - let mut message = CommitMessage::new( - BinaryRow::new(0).to_serialized_bytes(), - 0, - vec![data_file("data-0.parquet")], - ); - // Stored under the LEGACY identifier; request uses the canonical name. - message.new_index_files = vec![ - global_index_file("lumina-vector-ann", "lumina-id.index", 0, 0, 9), - global_index_file(BTREE_GLOBAL_INDEX_TYPE, "btree-id.index", 0, 0, 9), - ]; - TableCommit::new(table.clone(), "test-user".to_string()) - .commit(vec![message]) - .await - .unwrap(); - - let dropped = table - .new_global_index_drop_builder() - .with_index_column("id") - .with_index_type("lumina") - .execute() - .await - .unwrap(); - assert_eq!(dropped, 1); - - let mut remaining = latest_index_entries(&table) - .await - .into_iter() - .map(|entry| entry.index_file.file_name) - .collect::>(); - remaining.sort(); - assert_eq!(remaining, vec!["btree-id.index".to_string()]); - } - #[tokio::test] async fn test_drop_vindex_preserves_other_types_case_insensitive() { let table = test_table("memory:/test_drop_vindex_identity"); @@ -1243,7 +1204,7 @@ mod tests { message.new_index_files = vec![ global_index_file("ivf-flat", "ivf-flat-id.index", 0, 0, 9), global_index_file("ivf-pq", "ivf-pq-id.index", 0, 0, 9), - global_index_file("lumina", "lumina-id.index", 0, 0, 9), + global_index_file("diskann", "diskann-id.index", 0, 0, 9), global_index_file(BTREE_GLOBAL_INDEX_TYPE, "btree-id.index", 0, 0, 9), ]; TableCommit::new(table.clone(), "test-user".to_string()) @@ -1271,8 +1232,8 @@ mod tests { remaining, vec![ "btree-id.index".to_string(), + "diskann-id.index".to_string(), "ivf-pq-id.index".to_string(), - "lumina-id.index".to_string(), ] ); } diff --git a/crates/paimon/src/table/global_index_types.rs b/crates/paimon/src/table/global_index_types.rs index a51a54cc2..2ee944afb 100644 --- a/crates/paimon/src/table/global_index_types.rs +++ b/crates/paimon/src/table/global_index_types.rs @@ -15,7 +15,6 @@ // specific language governing permissions and limitations // under the License. -use crate::lumina::{is_lumina_index_type, LUMINA_IDENTIFIER}; use crate::vindex::{ is_vindex_index_type, DISKANN_IDENTIFIER, IVF_FLAT_IDENTIFIER, IVF_PQ_IDENTIFIER, IVF_RQ_IDENTIFIER, IVF_SQ_IDENTIFIER, @@ -52,14 +51,13 @@ pub(crate) fn normalize_queryable_global_index_type(index_type: &str) -> Option< /// Used verbatim in the unsupported-type error of both the builder and the /// DataFusion procedure so the two messages stay in sync. pub const SUPPORTED_GLOBAL_INDEX_TYPES_FOR_DROP: &str = - "btree, bitmap, multivalue, fm, full-text, lumina, lumina-vector-ann, ivf-flat, ivf-pq, ivf-sq, ivf-rq, diskann"; + "btree, bitmap, multivalue, fm, full-text, ivf-flat, ivf-pq, ivf-sq, ivf-rq, diskann"; /// Canonicalize any supported global index type to a stable `&'static str`, or /// `None` if unsupported. Case-insensitive and shared by build and drop. -/// Order: scalar -> full-text -> lumina -> vindex. +/// Order: scalar -> full-text -> vindex. /// -/// Both lumina aliases (`lumina`, `lumina-vector-ann`) canonicalize to -/// `"lumina"`; each vindex type keeps its own identity so dropping one vindex +/// Each vindex type keeps its own identity so dropping one vindex /// type never matches another on the same column. Callers should compare the /// canonical form of BOTH the request and each stored entry's `index_type`. pub fn normalize_global_index_type(index_type: &str) -> Option<&'static str> { @@ -69,12 +67,9 @@ pub fn normalize_global_index_type(index_type: &str) -> Option<&'static str> { if index_type.eq_ignore_ascii_case(FULL_TEXT_GLOBAL_INDEX_TYPE) { return Some(FULL_TEXT_GLOBAL_INDEX_TYPE); } - // is_lumina_index_type / is_vindex_index_type match case-sensitively, so + // is_vindex_index_type matches case-sensitively, so // lowercase first (normalize_sorted_global_index_type is already case-insensitive). let lowered = index_type.to_ascii_lowercase(); - if is_lumina_index_type(&lowered) { - return Some(LUMINA_IDENTIFIER); - } if is_vindex_index_type(&lowered) { return canonical_vindex_identifier(&lowered); } @@ -117,16 +112,6 @@ mod tests { assert_eq!(normalize_queryable_global_index_type("full-text"), None); } - #[test] - fn lumina_aliases_canonicalize_to_lumina() { - assert_eq!(normalize_global_index_type("lumina"), Some("lumina")); - assert_eq!( - normalize_global_index_type("lumina-vector-ann"), - Some("lumina") - ); - assert_eq!(normalize_global_index_type("LUMINA"), Some("lumina")); - } - #[test] fn vindex_types_keep_distinct_identity() { assert_eq!(normalize_global_index_type("ivf-flat"), Some("ivf-flat")); @@ -139,6 +124,8 @@ mod tests { #[test] fn unsupported_types_return_none() { assert_eq!(normalize_global_index_type("hash"), None); + assert_eq!(normalize_global_index_type("lumina"), None); + assert_eq!(normalize_global_index_type("lumina-vector-ann"), None); assert_eq!(normalize_global_index_type("ivf-hnsw-flat"), None); assert_eq!(normalize_global_index_type("ivf-hnsw-sq"), None); assert_eq!(normalize_global_index_type(""), None); diff --git a/crates/paimon/src/table/lumina_index_build_builder.rs b/crates/paimon/src/table/lumina_index_build_builder.rs deleted file mode 100644 index b96f43d94..000000000 --- a/crates/paimon/src/table/lumina_index_build_builder.rs +++ /dev/null @@ -1,177 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -mod extraction; -mod planning; -mod validation; -mod writer; - -use extraction::extract_vectors; -use planning::plan_lumina_shards; -use validation::{ - effective_lumina_options, find_index_field, resolve_lumina_options, validate_table_options, - validate_vector_field, -}; -use writer::abort_on_build_error; - -use crate::lumina::{LuminaIndexMeta, LuminaVectorIndexOptions, LUMINA_IDENTIFIER}; -use crate::spec::{CoreOptions, Predicate}; -use crate::table::{CommitMessage, RowRange, Table, TableCommit}; -use crate::{Error, Result}; -use std::collections::HashMap; - -pub(super) struct LuminaIndexBuildBuilder<'a> { - table: &'a Table, - index_column: Option, - options: HashMap, - pub(super) partition_filter: Option, -} - -impl<'a> LuminaIndexBuildBuilder<'a> { - pub(crate) fn new(table: &'a Table) -> Self { - Self { - table, - index_column: None, - options: HashMap::new(), - partition_filter: None, - } - } - - pub fn with_index_column(&mut self, column: &str) -> &mut Self { - self.index_column = Some(column.to_string()); - self - } - - pub fn with_options(&mut self, options: HashMap) -> &mut Self { - self.options = options; - self - } - - pub(super) async fn prepare(&self) -> Result<(Option, Vec)> { - // Building the index scans the table's rows. - CoreOptions::new(self.table.schema().options()).ensure_read_authorized()?; - - self.table.ensure_not_branch_reference_for_write()?; - - let index_column = self - .index_column - .as_deref() - .ok_or_else(|| Error::DataInvalid { - message: "Lumina index column is required".to_string(), - source: None, - })?; - - let mut merged_options = self.table.schema().options().clone(); - merged_options.extend(self.options.clone()); - let core_options = CoreOptions::new(&merged_options); - validate_table_options(self.table, &core_options)?; - let rows_per_shard = core_options.global_index_row_count_per_shard()?; - - let index_field = find_index_field(self.table, index_column)?; - validate_vector_field(index_field)?; - - let resolved_options = - resolve_lumina_options(self.table.schema().options(), &self.options)?; - let resolved_options = effective_lumina_options(index_field, resolved_options)?; - let lumina_options = LuminaVectorIndexOptions::new(&resolved_options)?; - let dimension = lumina_options.dimension; - let index_meta = LuminaIndexMeta::new(lumina_options.to_lumina_options()).serialize()?; - - let snapshot_manager = self.table.snapshot_manager(); - let Some(snapshot) = snapshot_manager.get_latest_snapshot().await? else { - return Ok((None, vec![])); - }; - - let mut read_builder = self.table.new_read_builder(); - if let Some(filter) = &self.partition_filter { - read_builder.with_filter(filter.clone()); - } - let manifest_entries = read_builder - .new_scan() - .with_scan_all_files() - .plan_manifest_entries(&snapshot) - .await?; - let indexed = crate::table::global_index_build_common::indexed_row_ranges( - self.table, - snapshot.index_manifest(), - LUMINA_IDENTIFIER, - index_field.id(), - None, // single-column build; no extra fields today - ) - .await?; - let shards = plan_lumina_shards( - self.table.location(), - self.table.schema().partition_keys(), - self.table.schema().fields(), - &core_options, - snapshot.id(), - manifest_entries, - rows_per_shard, - &indexed, - )?; - if shards.is_empty() { - return Ok((Some(snapshot.id()), vec![])); - } - - crate::table::global_index_build_common::validate_existing_index_overlap( - self.table, - snapshot.index_manifest(), - LUMINA_IDENTIFIER, - index_field.id(), - None, - &shards - .iter() - .map(|shard| RowRange::new(shard.row_range_start, shard.row_range_end)) - .collect::>(), - ) - .await?; - - let commit = TableCommit::new( - self.table.clone(), - format!( - "global-index-{}-create-{}", - LUMINA_IDENTIFIER, - uuid::Uuid::new_v4() - ), - ); - let shard_count = shards.len(); - let mut messages = Vec::with_capacity(shard_count); - for shard in shards { - let build_result = async { - let vectors = extract_vectors(self.table, &shard, index_column, dimension).await?; - self.build_index_file( - &shard, - &vectors, - dimension, - index_field.id(), - index_meta.clone(), - ) - .await - } - .await; - let index_file = abort_on_build_error(&commit, &messages, build_result).await?; - let mut message = CommitMessage::new(shard.partition_bytes.clone(), 0, vec![]); - message.new_index_files = vec![index_file]; - messages.push(message); - } - - Ok((Some(snapshot.id()), messages)) - } -} - -#[cfg(test)] -mod tests; diff --git a/crates/paimon/src/table/lumina_index_build_builder/extraction.rs b/crates/paimon/src/table/lumina_index_build_builder/extraction.rs deleted file mode 100644 index 7fb5b10fc..000000000 --- a/crates/paimon/src/table/lumina_index_build_builder/extraction.rs +++ /dev/null @@ -1,215 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -//! Arrow vector extraction for Lumina shards. - -use super::planning::LuminaIndexShard; -use super::validation::checked_row_count; -use crate::spec::ROW_ID_FIELD_NAME; -use crate::table::{DataSplitBuilder, RowRange, Table}; -use crate::{Error, Result}; -use arrow_array::{Array, FixedSizeListArray, Float32Array, Int64Array, ListArray, RecordBatch}; -use futures::TryStreamExt; - -pub(super) async fn extract_vectors( - table: &Table, - shard: &LuminaIndexShard, - index_column: &str, - dimension: i32, -) -> Result> { - let split = DataSplitBuilder::new() - .with_snapshot(shard.snapshot_id) - .with_partition(shard.partition.clone()) - .with_bucket(shard.source_bucket) - .with_bucket_path(shard.bucket_path.clone()) - .with_total_buckets(shard.total_buckets) - .with_data_files(shard.files.clone()) - .with_row_ranges(vec![RowRange::new( - shard.row_range_start, - shard.row_range_end, - )]) - .build()?; - - let mut read_builder = table.new_read_builder(); - read_builder.with_projection(&[index_column, ROW_ID_FIELD_NAME])?; - let read = read_builder.new_read()?; - let batches = read.to_arrow(&[split])?.try_collect::>().await?; - extract_vectors_from_batches( - &batches, - index_column, - dimension, - shard.row_range_start, - checked_row_count(shard.row_range_start, shard.row_range_end)?, - ) -} - -pub(super) fn extract_vectors_from_batches( - batches: &[RecordBatch], - index_column: &str, - dimension: i32, - row_range_start: i64, - expected_row_count: i64, -) -> Result> { - let dimension = usize::try_from(dimension).map_err(|e| Error::DataInvalid { - message: format!("Invalid Lumina dimension: {dimension}"), - source: Some(Box::new(e)), - })?; - let row_count = batches.iter().map(RecordBatch::num_rows).sum::(); - let mut vectors = Vec::with_capacity(row_count * dimension); - let mut expected_row_id = row_range_start; - for batch in batches { - let vector_index = - batch - .schema() - .index_of(index_column) - .map_err(|e| Error::DataInvalid { - message: format!("Vector column '{index_column}' not found in read batch: {e}"), - source: None, - })?; - let row_id_index = - batch - .schema() - .index_of(ROW_ID_FIELD_NAME) - .map_err(|e| Error::DataInvalid { - message: format!("_ROW_ID column not found in read batch: {e}"), - source: None, - })?; - // Resolve the vector column as either List (ARRAY) or - // FixedSizeList (VECTOR). Both yield a Float32Array of - // values plus a per-row [start, end) slice. - let column = batch.column(vector_index); - enum VectorLayout<'a> { - List(&'a ListArray), - Fixed(&'a FixedSizeListArray), - } - let layout = if let Some(a) = column.as_any().downcast_ref::() { - VectorLayout::List(a) - } else if let Some(a) = column.as_any().downcast_ref::() { - VectorLayout::Fixed(a) - } else { - return Err(Error::DataInvalid { - message: "Lumina vector extraction requires Arrow List or FixedSizeList".to_string(), - source: None, - }); - }; - let values = match layout { - VectorLayout::List(a) => a.values(), - VectorLayout::Fixed(a) => a.values(), - } - .as_any() - .downcast_ref::() - .ok_or_else(|| Error::DataInvalid { - message: "Lumina vector extraction requires Float32 vector elements".to_string(), - source: None, - })?; - let row_ids = batch - .column(row_id_index) - .as_any() - .downcast_ref::() - .ok_or_else(|| Error::DataInvalid { - message: "Lumina vector extraction requires non-null Int64 _ROW_ID".to_string(), - source: None, - })?; - - for row in 0..batch.num_rows() { - if row_ids.is_null(row) { - return Err(Error::DataInvalid { - message: "Lumina vector extraction found null _ROW_ID".to_string(), - source: None, - }); - } - let row_id = row_ids.value(row); - if row_id != expected_row_id { - return Err(Error::DataInvalid { - message: format!( - "Lumina vector extraction expected _ROW_ID {}, got {}", - expected_row_id, row_id - ), - source: None, - }); - } - expected_row_id += 1; - - let is_null = match layout { - VectorLayout::List(a) => a.is_null(row), - VectorLayout::Fixed(a) => a.is_null(row), - }; - if is_null { - return Err(Error::DataInvalid { - message: "Lumina vector extraction found null vector row".to_string(), - source: None, - }); - } - let (start, end) = match layout { - VectorLayout::List(a) => { - let offsets = a.value_offsets(); - (offsets[row] as usize, offsets[row + 1] as usize) - } - VectorLayout::Fixed(a) => { - let len = a.value_length() as usize; - (row * len, (row + 1) * len) - } - }; - if end - start != dimension { - return Err(Error::DataInvalid { - message: format!( - "Lumina vector dimension mismatch: expected {}, got {}", - dimension, - end - start - ), - source: None, - }); - } - for value_index in start..end { - if values.is_null(value_index) { - return Err(Error::DataInvalid { - message: "Lumina vector extraction found null vector element".to_string(), - source: None, - }); - } - let value = values.value(value_index); - // Java rejects these per element in `LuminaVectorGlobalIndexWriter`, and - // so does the vindex backend (`paimon_vindex_core`'s - // `validate_finite_values`). Without the check the index is built from - // undefined data and every later top-k over the shard is scored against - // it. - if !value.is_finite() { - return Err(Error::DataInvalid { - message: format!( - "Lumina vector extraction found non-finite element {value} \ - at _ROW_ID {row_id}, index {}", - value_index - start - ), - source: None, - }); - } - vectors.push(value); - } - } - } - let actual_row_count = expected_row_id - row_range_start; - if actual_row_count != expected_row_count { - return Err(Error::DataInvalid { - message: format!( - "Lumina vector extraction expected {} rows, got {}", - expected_row_count, actual_row_count - ), - source: None, - }); - } - Ok(vectors) -} diff --git a/crates/paimon/src/table/lumina_index_build_builder/planning.rs b/crates/paimon/src/table/lumina_index_build_builder/planning.rs deleted file mode 100644 index 0eafcbf88..000000000 --- a/crates/paimon/src/table/lumina_index_build_builder/planning.rs +++ /dev/null @@ -1,49 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -//! Lumina adapter for shared vector-index shard planning. - -use crate::spec::{CoreOptions, DataField, ManifestEntry}; -use crate::table::global_index_build_common::vector::{plan_vector_index_shards, VectorIndexShard}; -use crate::table::RowRange; -use crate::Result; - -pub(crate) type LuminaIndexShard = VectorIndexShard; - -#[allow(clippy::too_many_arguments)] -pub(super) fn plan_lumina_shards( - table_location: &str, - partition_keys: &[String], - schema_fields: &[DataField], - core_options: &CoreOptions, - snapshot_id: i64, - entries: Vec, - rows_per_shard: i64, - indexed: &[RowRange], -) -> Result> { - plan_vector_index_shards( - table_location, - partition_keys, - schema_fields, - core_options, - snapshot_id, - entries, - rows_per_shard, - indexed, - "Lumina", - ) -} diff --git a/crates/paimon/src/table/lumina_index_build_builder/tests.rs b/crates/paimon/src/table/lumina_index_build_builder/tests.rs deleted file mode 100644 index 43e5a93a4..000000000 --- a/crates/paimon/src/table/lumina_index_build_builder/tests.rs +++ /dev/null @@ -1,1316 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -use super::extraction::extract_vectors_from_batches; -use super::planning::{plan_lumina_shards, LuminaIndexShard}; -use super::validation::{ - checked_i64, effective_lumina_options, find_index_field, validate_vector_field, -}; -use super::writer::{abort_on_build_error, temp_lumina_path, TempFileGuard}; -use crate::catalog::Identifier; -use crate::io::FileIO; -use crate::io::FileIOBuilder; -use crate::lumina::{LUMINA_DIMENSION_OPTION, LUMINA_IDENTIFIER}; -use crate::spec::stats::BinaryTableStats; -use crate::spec::{ - ArrayType, BinaryRow, CoreOptions, DataField, DataFileMeta, DataType, DoubleType, FileKind, - FloatType, GlobalIndexMeta, IndexFileMeta, IndexManifest, IntType, ManifestEntry, Schema, - TableSchema, VectorType, ROW_ID_FIELD_NAME, -}; -use crate::table::source::exclude_row_ranges; -use crate::table::{CommitMessage, RowRange, SnapshotManager, Table, TableCommit, TableWrite}; -use crate::{Error, Result}; -use arrow_array::builder::{FixedSizeListBuilder, Float32Builder, Int64Builder, ListBuilder}; -use arrow_array::{ArrayRef, Int32Array, Int64Array, RecordBatch}; -use arrow_schema::{DataType as ArrowDataType, Field as ArrowField, Schema as ArrowSchema}; -use bytes::Bytes; -use chrono::{DateTime, Utc}; -use std::collections::HashMap; -use std::sync::Arc; - -fn data_file(name: &str, first_row_id: Option, row_count: i64) -> DataFileMeta { - DataFileMeta { - file_name: name.to_string(), - file_size: 128, - row_count, - min_key: vec![], - max_key: vec![], - key_stats: BinaryTableStats::new(vec![], vec![], vec![]), - value_stats: BinaryTableStats::new(vec![], vec![], vec![]), - min_sequence_number: 0, - max_sequence_number: 0, - schema_id: 0, - level: 0, - extra_files: vec![], - creation_time: Some( - "2024-09-06T07:45:55.039+00:00" - .parse::>() - .unwrap(), - ), - delete_row_count: None, - embedded_index: None, - first_row_id, - write_cols: None, - external_path: None, - file_source: None, - value_stats_cols: None, - column_max_sequence_numbers: None, - } -} - -fn manifest_entry(file: DataFileMeta) -> ManifestEntry { - manifest_entry_with_bucket(file, 0, 1) -} - -fn manifest_entry_with_bucket( - file: DataFileMeta, - bucket: i32, - total_buckets: i32, -) -> ManifestEntry { - ManifestEntry::new(FileKind::Add, vec![], bucket, total_buckets, file, 2) -} - -fn table_options(rows_per_shard: &str) -> HashMap { - HashMap::from([ - ("row-tracking.enabled".to_string(), "true".to_string()), - ("data-evolution.enabled".to_string(), "true".to_string()), - ("global-index.enabled".to_string(), "true".to_string()), - ( - "global-index.row-count-per-shard".to_string(), - rows_per_shard.to_string(), - ), - ]) -} - -fn test_table(options: HashMap) -> Table { - test_table_with_io( - FileIOBuilder::new("memory").build().unwrap(), - "memory:/test_lumina_builder", - Schema::builder() - .column("id", DataType::Int(IntType::new())) - .column( - "embedding", - DataType::Array(ArrayType::new(DataType::Float(FloatType::new()))), - ) - .options(options) - .build() - .unwrap(), - ) -} - -fn test_table_with_schema(schema: Schema) -> Table { - test_table_with_io( - FileIOBuilder::new("memory").build().unwrap(), - "memory:/test_lumina_builder", - schema, - ) -} - -fn test_table_with_io(file_io: FileIO, table_path: &str, schema: Schema) -> Table { - Table::new( - file_io, - Identifier::new("default", "test_table"), - table_path.to_string(), - TableSchema::new(0, &schema), - None, - ) -} - -fn vector_schema_builder(options: HashMap) -> crate::spec::SchemaBuilder { - Schema::builder() - .column("id", DataType::Int(IntType::new())) - .column( - "embedding", - DataType::Array(ArrayType::new(DataType::Float(FloatType::new()))), - ) - .options(options) -} - -fn plan(entries: Vec, rows_per_shard: i64) -> Result> { - plan_with_indexed(entries, rows_per_shard, &[]) -} - -fn plan_with_indexed( - entries: Vec, - rows_per_shard: i64, - indexed: &[RowRange], -) -> Result> { - let table = test_table(table_options(&rows_per_shard.to_string())); - let core = CoreOptions::new(table.schema().options()); - plan_lumina_shards( - table.location(), - table.schema().partition_keys(), - table.schema().fields(), - &core, - 1, - entries, - rows_per_shard, - indexed, - ) -} - -#[test] -fn test_planner_splits_single_file_across_shards() { - let shards = plan(vec![manifest_entry(data_file("a", Some(0), 25))], 10).unwrap(); - - assert_eq!( - shards - .iter() - .map(|s| (s.row_range_start, s.row_range_end)) - .collect::>(), - vec![(0, 9), (10, 19), (20, 24)] - ); -} - -#[test] -fn test_planner_merges_contiguous_files() { - let shards = plan( - vec![ - manifest_entry(data_file("a", Some(0), 5)), - manifest_entry(data_file("b", Some(5), 5)), - ], - 20, - ) - .unwrap(); - - assert_eq!(shards.len(), 1); - assert_eq!((shards[0].row_range_start, shards[0].row_range_end), (0, 9)); - assert_eq!( - shards[0] - .files - .iter() - .map(|f| f.file_name.as_str()) - .collect::>(), - vec!["a", "b"] - ); -} - -#[test] -fn test_planner_keeps_source_buckets_separate() { - let shards = plan( - vec![ - manifest_entry_with_bucket(data_file("a", Some(0), 5), 0, 2), - manifest_entry_with_bucket(data_file("b", Some(5), 5), 1, 2), - ], - 20, - ) - .unwrap(); - - assert_eq!(shards.len(), 2); - assert_eq!( - shards - .iter() - .map(|s| ( - s.source_bucket, - s.total_buckets, - s.row_range_start, - s.row_range_end - )) - .collect::>(), - vec![(0, 2, 0, 4), (1, 2, 5, 9)] - ); -} - -#[test] -fn test_planner_splits_gap_into_separate_groups() { - let shards = plan( - vec![ - manifest_entry(data_file("a", Some(0), 5)), - manifest_entry(data_file("b", Some(10), 5)), - ], - 20, - ) - .unwrap(); - - assert_eq!( - shards - .iter() - .map(|s| (s.row_range_start, s.row_range_end)) - .collect::>(), - vec![(0, 4), (10, 14)] - ); -} - -#[test] -fn test_planner_rejects_missing_first_row_id() { - let err = plan(vec![manifest_entry(data_file("a", None, 5))], 10) - .expect_err("missing first_row_id should fail"); - assert!( - matches!(err, Error::DataInvalid { message, .. } if message.contains("missing first_row_id")) - ); -} - -#[test] -fn test_planner_rejects_invalid_rows_per_shard() { - let err = plan(vec![manifest_entry(data_file("a", Some(0), 5))], 0) - .expect_err("invalid rows per shard should fail"); - assert!( - matches!(err, Error::DataInvalid { message, .. } if message.contains("row-count-per-shard")) - ); -} - -#[test] -fn test_validate_vector_field_accepts_array_float() { - let field = DataField::new( - 0, - "embedding".to_string(), - DataType::Array(ArrayType::new(DataType::Float(FloatType::new()))), - ); - assert!(validate_vector_field(&field).is_ok()); -} - -#[test] -fn test_validate_vector_field_accepts_vector_float() { - let field = DataField::new( - 0, - "embedding".to_string(), - DataType::Vector(VectorType::try_new(true, 4, DataType::Float(FloatType::new())).unwrap()), - ); - assert!(validate_vector_field(&field).is_ok()); -} - -#[test] -fn test_validate_vector_field_rejects_vector_double() { - let field = DataField::new( - 0, - "embedding".to_string(), - DataType::Vector( - VectorType::try_new(true, 4, DataType::Double(DoubleType::new())).unwrap(), - ), - ); - let err = validate_vector_field(&field).expect_err("VECTOR must be rejected"); - assert!(matches!(err, Error::DataInvalid { .. })); -} - -#[test] -fn test_effective_options_vector_absent_inserts_length() { - let field = DataField::new( - 0, - "embedding".to_string(), - DataType::Vector( - VectorType::try_new(true, 256, DataType::Float(FloatType::new())).unwrap(), - ), - ); - let opts = effective_lumina_options(&field, HashMap::new()).unwrap(); - assert_eq!( - opts.get(LUMINA_DIMENSION_OPTION).map(String::as_str), - Some("256") - ); -} - -#[test] -fn test_effective_options_vector_matching_option_ok() { - let field = DataField::new( - 0, - "embedding".to_string(), - DataType::Vector( - VectorType::try_new(true, 256, DataType::Float(FloatType::new())).unwrap(), - ), - ); - let resolved = HashMap::from([(LUMINA_DIMENSION_OPTION.to_string(), "256".to_string())]); - let opts = effective_lumina_options(&field, resolved).unwrap(); - assert_eq!( - opts.get(LUMINA_DIMENSION_OPTION).map(String::as_str), - Some("256") - ); -} - -#[test] -fn test_effective_options_vector_mismatch_errors() { - let field = DataField::new( - 0, - "embedding".to_string(), - DataType::Vector( - VectorType::try_new(true, 256, DataType::Float(FloatType::new())).unwrap(), - ), - ); - let resolved = HashMap::from([(LUMINA_DIMENSION_OPTION.to_string(), "128".to_string())]); - let err = - effective_lumina_options(&field, resolved).expect_err("dimension mismatch must error"); - assert!(matches!(err, Error::ConfigInvalid { .. })); -} - -#[test] -fn test_effective_options_array_unchanged() { - let field = DataField::new( - 0, - "embedding".to_string(), - DataType::Array(ArrayType::new(DataType::Float(FloatType::new()))), - ); - // No dimension option set: array path must NOT inject one. - let opts = effective_lumina_options(&field, HashMap::new()).unwrap(); - assert!(!opts.contains_key(LUMINA_DIMENSION_OPTION)); -} - -#[test] -fn test_vector_without_option_propagates_dimension_to_native_options() { - use crate::lumina::{LuminaVectorIndexOptions, KEY_DIMENSION}; - let field = DataField::new( - 0, - "embedding".to_string(), - DataType::Vector( - VectorType::try_new(true, 256, DataType::Float(FloatType::new())).unwrap(), - ), - ); - // No lumina.index.dimension set by the user. - let resolved = effective_lumina_options(&field, HashMap::new()).unwrap(); - let opts = LuminaVectorIndexOptions::new(&resolved).unwrap(); - - assert_eq!( - opts.dimension, 256, - "local dimension must be N, not default 128" - ); - assert_eq!( - opts.to_lumina_options() - .get(KEY_DIMENSION) - .map(String::as_str), - Some("256"), - "native index.dimension must be N, not default 128" - ); -} - -#[tokio::test] -async fn test_execute_rejects_primary_key_table() { - let table = test_table_with_schema( - vector_schema_builder(HashMap::new()) - .primary_key(["id"]) - .build() - .unwrap(), - ); - - let err = table - .new_global_index_build_builder() - .with_index_type("lumina") - .with_index_column("embedding") - .execute() - .await - .expect_err("primary-key table should fail before native build"); - - assert!( - matches!(err, Error::Unsupported { message } if message.contains("primary-key tables")) - ); -} - -#[tokio::test] -async fn test_execute_rejects_deletion_vectors_table() { - let mut options = table_options("10"); - options.insert("deletion-vectors.enabled".to_string(), "true".to_string()); - let table = test_table(options); - - let err = table - .new_global_index_build_builder() - .with_index_type("lumina") - .with_index_column("embedding") - .execute() - .await - .expect_err("deletion vectors table should fail before native build"); - - assert!( - matches!(err, Error::Unsupported { message } if message.contains("deletion-vectors.enabled=true")) - ); -} - -fn vector_batch(rows: Vec>>>, row_ids: Vec>) -> RecordBatch { - let mut vector_builder = ListBuilder::new(Float32Builder::new()); - for row in rows { - match row { - Some(values) => { - for value in values { - match value { - Some(value) => vector_builder.values().append_value(value), - None => vector_builder.values().append_null(), - } - } - vector_builder.append(true); - } - None => vector_builder.append(false), - } - } - let mut row_id_builder = Int64Builder::new(); - for row_id in row_ids { - match row_id { - Some(value) => row_id_builder.append_value(value), - None => row_id_builder.append_null(), - } - } - let schema = Arc::new(ArrowSchema::new(vec![ - ArrowField::new( - "embedding", - ArrowDataType::List(Arc::new(ArrowField::new( - "item", - ArrowDataType::Float32, - true, - ))), - true, - ), - ArrowField::new(ROW_ID_FIELD_NAME, ArrowDataType::Int64, true), - ])); - RecordBatch::try_new( - schema, - vec![ - Arc::new(vector_builder.finish()) as ArrayRef, - Arc::new(row_id_builder.finish()) as ArrayRef, - ], - ) - .unwrap() -} - -#[test] -fn test_extract_vectors_accepts_list_float32_and_row_ids() { - let batch = vector_batch( - vec![ - Some(vec![Some(1.0), Some(2.0)]), - Some(vec![Some(3.0), Some(4.0)]), - ], - vec![Some(10), Some(11)], - ); - - let vectors = extract_vectors_from_batches(&[batch], "embedding", 2, 10, 2).unwrap(); - - assert_eq!(vectors, vec![1.0, 2.0, 3.0, 4.0]); -} - -#[test] -fn test_extract_vectors_rejects_null_vector() { - let batch = vector_batch(vec![None], vec![Some(0)]); - - let err = extract_vectors_from_batches(&[batch], "embedding", 2, 0, 1) - .expect_err("null vector should fail"); - - assert!(matches!(err, Error::DataInvalid { message, .. } if message.contains("null vector"))); -} - -#[test] -fn test_extract_vectors_rejects_null_element() { - let batch = vector_batch(vec![Some(vec![Some(1.0), None])], vec![Some(0)]); - - let err = extract_vectors_from_batches(&[batch], "embedding", 2, 0, 1) - .expect_err("null element should fail"); - - assert!( - matches!(err, Error::DataInvalid { message, .. } if message.contains("null vector element")) - ); -} - -#[test] -fn test_extract_vectors_rejects_dimension_mismatch() { - let batch = vector_batch(vec![Some(vec![Some(1.0)])], vec![Some(0)]); - - let err = extract_vectors_from_batches(&[batch], "embedding", 2, 0, 1) - .expect_err("dimension mismatch should fail"); - - assert!( - matches!(err, Error::DataInvalid { message, .. } if message.contains("dimension mismatch")) - ); -} - -#[test] -fn test_extract_vectors_rejects_row_id_gap() { - let batch = vector_batch( - vec![ - Some(vec![Some(1.0), Some(2.0)]), - Some(vec![Some(3.0), Some(4.0)]), - ], - vec![Some(0), Some(2)], - ); - - let err = extract_vectors_from_batches(&[batch], "embedding", 2, 0, 2) - .expect_err("row id gap should fail"); - - assert!( - matches!(err, Error::DataInvalid { message, .. } if message.contains("expected _ROW_ID")) - ); -} - -#[test] -fn test_extract_vectors_rejects_non_list_float32() { - let schema = Arc::new(ArrowSchema::new(vec![ - ArrowField::new("embedding", ArrowDataType::Int32, false), - ArrowField::new(ROW_ID_FIELD_NAME, ArrowDataType::Int64, true), - ])); - let batch = RecordBatch::try_new( - schema, - vec![ - Arc::new(Int32Array::from(vec![1])) as ArrayRef, - Arc::new(Int64Array::from(vec![Some(0)])) as ArrayRef, - ], - ) - .unwrap(); - - let err = extract_vectors_from_batches(&[batch], "embedding", 2, 0, 1) - .expect_err("non-list vector should fail"); - - assert!(matches!(err, Error::DataInvalid { message, .. } if message.contains("List"))); -} - -#[test] -fn test_extract_vectors_rejects_non_finite_element() { - for value in [f32::NAN, f32::INFINITY, f32::NEG_INFINITY] { - let batch = vector_batch(vec![Some(vec![Some(1.0), Some(value)])], vec![Some(7)]); - - let err = extract_vectors_from_batches(&[batch], "embedding", 2, 7, 1) - .expect_err("a non-finite element should fail"); - - let Error::DataInvalid { message, .. } = err else { - panic!("expected DataInvalid for {value}, got {err:?}"); - }; - assert!(message.contains("non-finite"), "message: {message}"); - // The row id and the position inside the vector are what make the failure - // actionable, as in Java's `Vector element at rowId=%d, index=%d`. - assert!(message.contains("_ROW_ID 7"), "message: {message}"); - assert!(message.contains("index 1"), "message: {message}"); - } -} - -fn fixed_size_vector_batch( - rows: Vec>>, - row_ids: Vec>, - len: i32, -) -> RecordBatch { - let element_field = Arc::new(ArrowField::new("element", ArrowDataType::Float32, true)); - let mut builder = - FixedSizeListBuilder::new(Float32Builder::new(), len).with_field(element_field); - for row in rows { - match row { - Some(values) => { - for v in values { - builder.values().append_value(v); - } - builder.append(true); - } - None => { - for _ in 0..len { - builder.values().append_value(0.0); - } - builder.append(false); - } - } - } - let mut row_id_builder = Int64Builder::new(); - for row_id in row_ids { - match row_id { - Some(value) => row_id_builder.append_value(value), - None => row_id_builder.append_null(), - } - } - let schema = Arc::new(ArrowSchema::new(vec![ - ArrowField::new( - "embedding", - ArrowDataType::FixedSizeList( - Arc::new(ArrowField::new("element", ArrowDataType::Float32, true)), - len, - ), - true, - ), - ArrowField::new(ROW_ID_FIELD_NAME, ArrowDataType::Int64, true), - ])); - RecordBatch::try_new( - schema, - vec![ - Arc::new(builder.finish()) as ArrayRef, - Arc::new(row_id_builder.finish()) as ArrayRef, - ], - ) - .unwrap() -} - -#[test] -fn test_extract_vectors_accepts_fixed_size_list_float32() { - let batch = fixed_size_vector_batch( - vec![Some(vec![1.0, 2.0]), Some(vec![3.0, 4.0])], - vec![Some(10), Some(11)], - 2, - ); - let vectors = extract_vectors_from_batches(&[batch], "embedding", 2, 10, 2).unwrap(); - assert_eq!(vectors, vec![1.0, 2.0, 3.0, 4.0]); -} - -#[test] -fn test_extract_vectors_fixed_size_list_rejects_null_vector() { - let batch = fixed_size_vector_batch(vec![None], vec![Some(0)], 2); - let err = extract_vectors_from_batches(&[batch], "embedding", 2, 0, 1) - .expect_err("null vector should fail"); - assert!(matches!(err, Error::DataInvalid { message, .. } if message.contains("null vector"))); -} - -#[test] -fn test_extract_vectors_fixed_size_list_dimension_mismatch() { - // Column is FixedSizeList of length 3, but caller expects dimension 2. - let batch = fixed_size_vector_batch(vec![Some(vec![1.0, 2.0, 3.0])], vec![Some(0)], 3); - let err = extract_vectors_from_batches(&[batch], "embedding", 2, 0, 1) - .expect_err("dimension mismatch should fail"); - assert!( - matches!(err, Error::DataInvalid { message, .. } if message.contains("dimension mismatch")) - ); -} - -#[test] -fn test_extract_vectors_fixed_size_list_rejects_null_element() { - // A non-null vector row whose second child element is null. Mirrors the - // List path's test_extract_vectors_rejects_null_element so both layouts - // reject null elements identically. - let element_field = Arc::new(ArrowField::new("element", ArrowDataType::Float32, true)); - let mut builder = FixedSizeListBuilder::new(Float32Builder::new(), 2).with_field(element_field); - builder.values().append_value(1.0); - builder.values().append_null(); - builder.append(true); - let row_ids = Arc::new(Int64Array::from(vec![Some(0)])) as ArrayRef; - let schema = Arc::new(ArrowSchema::new(vec![ - ArrowField::new( - "embedding", - ArrowDataType::FixedSizeList( - Arc::new(ArrowField::new("element", ArrowDataType::Float32, true)), - 2, - ), - true, - ), - ArrowField::new(ROW_ID_FIELD_NAME, ArrowDataType::Int64, true), - ])); - let batch = RecordBatch::try_new( - schema, - vec![Arc::new(builder.finish()) as ArrayRef, row_ids], - ) - .unwrap(); - - let err = extract_vectors_from_batches(&[batch], "embedding", 2, 0, 1) - .expect_err("null element should fail"); - - assert!( - matches!(err, Error::DataInvalid { message, .. } if message.contains("null vector element")) - ); -} - -#[test] -fn test_extract_vectors_fixed_size_list_rejects_non_finite_element() { - // Both layouts share the element loop, so this pins that the FixedSizeList - // path reaches it too. - let element_field = Arc::new(ArrowField::new("element", ArrowDataType::Float32, true)); - let mut builder = FixedSizeListBuilder::new(Float32Builder::new(), 2).with_field(element_field); - builder.values().append_value(1.0); - builder.values().append_value(f32::NAN); - builder.append(true); - let row_ids = Arc::new(Int64Array::from(vec![Some(0)])) as ArrayRef; - let schema = Arc::new(ArrowSchema::new(vec![ - ArrowField::new( - "embedding", - ArrowDataType::FixedSizeList( - Arc::new(ArrowField::new("element", ArrowDataType::Float32, true)), - 2, - ), - true, - ), - ArrowField::new(ROW_ID_FIELD_NAME, ArrowDataType::Int64, true), - ])); - let batch = RecordBatch::try_new( - schema, - vec![Arc::new(builder.finish()) as ArrayRef, row_ids], - ) - .unwrap(); - - let err = extract_vectors_from_batches(&[batch], "embedding", 2, 0, 1) - .expect_err("a non-finite element should fail"); - - assert!(matches!(err, Error::DataInvalid { message, .. } if message.contains("non-finite"))); -} - -#[test] -fn test_checked_metadata_conversion_supports_i64_file_size() { - let above_i32_max = i32::MAX as u64 + 1; - assert_eq!( - checked_i64(above_i32_max, "Index file is too large").unwrap(), - i64::from(i32::MAX) + 1 - ); - - let err = checked_i64(i64::MAX as u64 + 1, "Index file is too large") - .expect_err("file size above i64::MAX should fail"); - assert!(matches!(err, Error::DataInvalid { message, .. } if message.contains("too large"))); -} - -#[test] -fn test_temp_file_guard_cleans_up_on_drop() { - let path = temp_lumina_path(); - std::fs::write(&path, b"temporary lumina data").unwrap(); - { - let _guard = TempFileGuard::new(path.clone()); - assert!(path.exists()); - } - assert!(!path.exists()); -} - -#[tokio::test] -async fn test_abort_on_build_error_removes_completed_index_files() { - let file_io = FileIOBuilder::new("memory").build().unwrap(); - let table_path = "memory:/test_lumina_abort_cleanup"; - let table = test_table_with_io( - file_io.clone(), - table_path, - vector_schema_builder(table_options("1")).build().unwrap(), - ); - file_io - .mkdirs(&format!("{table_path}/index/")) - .await - .unwrap(); - - let file_name = "completed-lumina-shard.index"; - let index_path = format!("{table_path}/index/{file_name}"); - file_io - .new_output(&index_path) - .unwrap() - .write(Bytes::from_static(b"completed shard")) - .await - .unwrap(); - let mut message = CommitMessage::new(vec![], 0, vec![]); - message.new_index_files = vec![IndexFileMeta { - index_type: LUMINA_IDENTIFIER.to_string(), - file_name: file_name.to_string(), - file_size: 15, - row_count: 1, - deletion_vectors_ranges: None, - external_path: None, - global_index_meta: None, - }]; - - let commit = TableCommit::new(table, "test-lumina-abort".to_string()); - let result = abort_on_build_error::<()>( - &commit, - &[message], - Err(Error::UnexpectedError { - message: "second shard failed".to_string(), - source: None, - }), - ) - .await; - - assert!(result.is_err()); - assert!(file_io.get_status(&index_path).await.is_err()); -} - -async fn setup_dirs(file_io: &FileIO, table_path: &str) { - file_io - .mkdirs(&format!("{table_path}/snapshot/")) - .await - .unwrap(); - file_io - .mkdirs(&format!("{table_path}/manifest/")) - .await - .unwrap(); -} - -fn build_vector_batch(ids: Vec, vectors: Vec>) -> RecordBatch { - let element_field = Arc::new(ArrowField::new("element", ArrowDataType::Float32, true)); - let mut vector_builder = - ListBuilder::new(Float32Builder::new()).with_field(element_field.clone()); - for vector in vectors { - for value in vector { - vector_builder.values().append_value(value); - } - vector_builder.append(true); - } - let schema = Arc::new(ArrowSchema::new(vec![ - ArrowField::new("id", ArrowDataType::Int32, false), - ArrowField::new("embedding", ArrowDataType::List(element_field), true), - ])); - RecordBatch::try_new( - schema, - vec![ - Arc::new(Int32Array::from(ids)) as ArrayRef, - Arc::new(vector_builder.finish()) as ArrayRef, - ], - ) - .unwrap() -} - -// Manual run with a local Lumina native library: -// LUMINA_LIB_PATH=/path/to/liblumina_py.so cargo test -p paimon \ -// table::lumina_index_build_builder::tests::test_execute_writes_lumina_index_manifest \ -// --features fulltext,vortex -- --ignored --exact -#[tokio::test] -#[ignore = "requires LUMINA_LIB_PATH; see manual run command above"] -async fn test_execute_writes_lumina_index_manifest() { - let file_io = FileIOBuilder::new("memory").build().unwrap(); - let table_path = "memory:/test_lumina_builder_e2e"; - setup_dirs(&file_io, table_path).await; - - let mut options = table_options("10"); - options.insert("lumina.index.dimension".to_string(), "2".to_string()); - options.insert("lumina.encoding.type".to_string(), "rawf32".to_string()); - let table = test_table_with_io( - file_io.clone(), - table_path, - vector_schema_builder(options).build().unwrap(), - ); - - let mut table_write = TableWrite::new(&table, "test-user".to_string()).unwrap(); - table_write - .write_arrow_batch(&build_vector_batch( - vec![1, 2], - vec![vec![1.0, 0.0], vec![0.0, 1.0]], - )) - .await - .unwrap(); - let messages = table_write.prepare_commit().await.unwrap(); - TableCommit::new(table.clone(), "test-user".to_string()) - .commit(messages) - .await - .unwrap(); - - let shard_count = table - .new_global_index_build_builder() - .with_index_type("lumina") - .with_index_column("embedding") - .execute() - .await - .unwrap(); - assert_eq!(shard_count, 1); - - let snapshot_manager = SnapshotManager::new(file_io.clone(), table_path.to_string()); - let snapshot = snapshot_manager - .get_latest_snapshot() - .await - .unwrap() - .unwrap(); - assert_eq!(snapshot.id(), 2); - assert_eq!(snapshot.total_record_count(), Some(2)); - assert_eq!(snapshot.delta_record_count(), Some(0)); - assert_eq!(snapshot.next_row_id(), Some(2)); - - let index_manifest = snapshot.index_manifest().expect("index manifest"); - let index_entries = - IndexManifest::read(&file_io, &format!("{table_path}/manifest/{index_manifest}")) - .await - .unwrap(); - assert_eq!(index_entries.len(), 1); - - let index_file = &index_entries[0].index_file; - assert_eq!(index_file.index_type, LUMINA_IDENTIFIER); - assert!(index_file.file_name.starts_with("lumina-global-index-")); - assert_eq!(index_file.row_count, 2); - assert!(index_file.file_size > 0); - - let global_meta = index_file - .global_index_meta - .as_ref() - .expect("global index meta"); - assert_eq!(global_meta.row_range_start, 0); - assert_eq!(global_meta.row_range_end, 1); - assert_eq!(global_meta.index_field_id, 1); - assert!(global_meta - .index_meta - .as_ref() - .is_some_and(|m| !m.is_empty())); - - let index_path = format!("{table_path}/index/{}", index_file.file_name); - let status = file_io.get_status(&index_path).await.unwrap(); - assert_eq!(index_file.file_size as u64, status.size); -} - -fn lumina_e2e_options(rows_per_shard: &str) -> HashMap { - let mut options = table_options(rows_per_shard); - options.insert("lumina.index.dimension".to_string(), "2".to_string()); - options.insert("lumina.encoding.type".to_string(), "rawf32".to_string()); - options -} - -fn lumina_e2e_table(table_path: &str, rows_per_shard: &str) -> Table { - test_table_with_io( - FileIOBuilder::new("memory").build().unwrap(), - table_path, - vector_schema_builder(lumina_e2e_options(rows_per_shard)) - .build() - .unwrap(), - ) -} - -async fn write_vectors(table: &Table, ids: Vec, vectors: Vec>) { - let mut table_write = TableWrite::new(table, "test-user".to_string()).unwrap(); - table_write - .write_arrow_batch(&build_vector_batch(ids, vectors)) - .await - .unwrap(); - let messages = table_write.prepare_commit().await.unwrap(); - TableCommit::new(table.clone(), "test-user".to_string()) - .commit(messages) - .await - .unwrap(); -} - -/// Commit a synthetic Lumina `IndexFileMeta` covering `[start, end]` for -/// `field_id` directly into the index manifest, without invoking the native -/// builder. Mirrors the btree mid-hole test so the incremental gap logic can -/// be exercised in CI where the native Lumina library is unavailable. -async fn commit_synthetic_lumina_index(table: &Table, field_id: i32, start: i64, end: i64) { - let synthetic = IndexFileMeta { - index_type: LUMINA_IDENTIFIER.to_string(), - file_name: format!("lumina-synthetic-{start}-{end}.index"), - file_size: 1, - row_count: (end - start + 1), - deletion_vectors_ranges: None, - external_path: None, - global_index_meta: Some(GlobalIndexMeta { - row_range_start: start, - row_range_end: end, - index_field_id: field_id, - extra_field_ids: None, - source_meta: None, - index_meta: None, - }), - }; - let mut message = CommitMessage::new(BinaryRow::new(0).to_serialized_bytes(), 0, vec![]); - message.new_index_files = vec![synthetic]; - TableCommit::new(table.clone(), "test-user".to_string()) - .commit(vec![message]) - .await - .unwrap(); -} - -async fn latest_lumina_index_files(table: &Table) -> Vec { - let snapshot_manager = - SnapshotManager::new(table.file_io().clone(), table.location().to_string()); - let snapshot = snapshot_manager - .get_latest_snapshot() - .await - .unwrap() - .unwrap(); - let Some(index_manifest_name) = snapshot.index_manifest() else { - return Vec::new(); - }; - IndexManifest::read( - table.file_io(), - &snapshot_manager.manifest_path(index_manifest_name), - ) - .await - .unwrap() - .into_iter() - .filter(|entry| entry.kind == FileKind::Add && entry.index_file.index_type == LUMINA_IDENTIFIER) - .map(|entry| entry.index_file) - .collect() -} - -/// Row-id coverage of the committed data files, read back from the data -/// manifest (never hard-coded) and merged into contiguous ranges. Mirrors -/// how `execute` gathers `manifest_entries`. -async fn data_row_id_coverage(table: &Table) -> Vec { - let snapshot_manager = - SnapshotManager::new(table.file_io().clone(), table.location().to_string()); - let snapshot = snapshot_manager - .get_latest_snapshot() - .await - .unwrap() - .unwrap(); - let entries = table - .new_read_builder() - .new_scan() - .with_scan_all_files() - .plan_manifest_entries(&snapshot) - .await - .unwrap(); - let ranges = entries - .iter() - .filter(|entry| *entry.kind() == FileKind::Add) - .filter_map(|entry| { - entry - .file() - .row_id_range() - .map(|(start, end)| RowRange::new(start, end)) - }) - .collect::>(); - crate::table::merge_row_ranges(ranges) -} - -/// Second build with the whole coverage already indexed must be a clean -/// no-op (returns 0), not an overlap error. Reaches `Ok(0)` before the -/// native build, so it runs in CI without the Lumina library. This is the -/// core bug fix: today the second call errors with the overlap message. -#[tokio::test] -async fn lumina_second_build_without_new_data_is_noop() { - let table_path = "memory:/test_lumina_second_build_noop"; - let table = lumina_e2e_table(table_path, "10"); - setup_dirs(table.file_io(), table_path).await; - - write_vectors( - &table, - vec![1, 2, 3], - vec![vec![1.0, 0.0], vec![0.0, 1.0], vec![1.0, 1.0]], - ) - .await; - - // Fully index the coverage via a synthetic manifest entry. - let coverage = data_row_id_coverage(&table).await; - assert_eq!(coverage.len(), 1, "data must be one contiguous range"); - let field_id = find_index_field(&table, "embedding").unwrap().id(); - commit_synthetic_lumina_index(&table, field_id, coverage[0].from(), coverage[0].to()).await; - - let names_before = latest_lumina_index_files(&table) - .await - .iter() - .map(|f| f.file_name.clone()) - .collect::>(); - assert!(!names_before.is_empty()); - - let built = table - .new_global_index_build_builder() - .with_index_type("lumina") - .with_index_column("embedding") - .execute() - .await - .unwrap(); - assert_eq!(built, 0, "fully-indexed table must build nothing on re-run"); - - let names_after = latest_lumina_index_files(&table) - .await - .iter() - .map(|f| f.file_name.clone()) - .collect::>(); - assert_eq!( - names_before, names_after, - "re-run must not add or remove index manifest entries" - ); -} - -/// Build over an already-indexed prefix, then append new rows: the second -/// build must target only the appended gap and must NOT fail with the -/// overlap error. Without the native Lumina library the gap build surfaces a -/// library-load error (not the overlap error); with it present it succeeds. -#[tokio::test] -async fn lumina_incremental_build_indexes_only_new_rows() { - let table_path = "memory:/test_lumina_incremental"; - let table = lumina_e2e_table(table_path, "10"); - setup_dirs(table.file_io(), table_path).await; - - // Initial batch, then mark it fully indexed via a synthetic entry. - write_vectors( - &table, - vec![1, 2, 3], - vec![vec![1.0, 0.0], vec![0.0, 1.0], vec![1.0, 1.0]], - ) - .await; - let indexed_coverage = data_row_id_coverage(&table).await; - assert_eq!(indexed_coverage.len(), 1); - let n = indexed_coverage[0].to() + 1; - let field_id = find_index_field(&table, "embedding").unwrap().id(); - commit_synthetic_lumina_index( - &table, - field_id, - indexed_coverage[0].from(), - indexed_coverage[0].to(), - ) - .await; - - // Append a second batch (new row-ids [n..]). - write_vectors( - &table, - vec![4, 5, 6], - vec![vec![2.0, 0.0], vec![0.0, 2.0], vec![2.0, 2.0]], - ) - .await; - - // White-box: fed the real indexed ranges from the manifest, the planner - // must target only the appended gap [n, ..], never the already-indexed - // prefix. Computed before `execute` so it is independent of whether the - // native build (which needs the Lumina library) runs. - let snapshot_manager = - SnapshotManager::new(table.file_io().clone(), table.location().to_string()); - let snapshot = snapshot_manager - .get_latest_snapshot() - .await - .unwrap() - .unwrap(); - let manifest_entries = table - .new_read_builder() - .new_scan() - .with_scan_all_files() - .plan_manifest_entries(&snapshot) - .await - .unwrap(); - let indexed = crate::table::global_index_build_common::indexed_row_ranges( - &table, - snapshot.index_manifest(), - LUMINA_IDENTIFIER, - field_id, - None, - ) - .await - .unwrap(); - let core = CoreOptions::new(table.schema().options()); - let shards = plan_lumina_shards( - table.location(), - table.schema().partition_keys(), - table.schema().fields(), - &core, - snapshot.id(), - manifest_entries, - 10, - &indexed, - ) - .unwrap(); - assert!(!shards.is_empty(), "appended gap must produce build shards"); - for shard in &shards { - assert!( - shard.row_range_start >= n, - "shard [{}, {}] must start at or after the indexed prefix end {n}", - shard.row_range_start, - shard.row_range_end - ); - } - - // End-to-end: the incremental build must no longer fail with the overlap - // error. Without the native Lumina library the gap build surfaces a - // library-load error instead; with it present it succeeds. - let result = table - .new_global_index_build_builder() - .with_index_type("lumina") - .with_index_column("embedding") - .execute() - .await; - match result { - Ok(_) => {} - Err(Error::DataInvalid { message, .. }) => { - assert!( - !message.contains("overlaps requested row range"), - "incremental build must not fail with the overlap error; got: {message}" - ); - } - Err(other) => panic!("unexpected error from incremental build: {other:?}"), - } -} - -/// Regression: a first build (no existing index) must equal the pre-change -/// full build -- subtracting an empty `indexed` yields full coverage. -#[test] -fn lumina_first_build_indexes_full_coverage() { - let full = plan(vec![manifest_entry(data_file("a", Some(0), 25))], 10).unwrap(); - let gapped = - plan_with_indexed(vec![manifest_entry(data_file("a", Some(0), 25))], 10, &[]).unwrap(); - // Empty `indexed` must not alter the shard layout. - assert_eq!( - full.iter() - .map(|s| (s.row_range_start, s.row_range_end)) - .collect::>(), - gapped - .iter() - .map(|s| (s.row_range_start, s.row_range_end)) - .collect::>() - ); - assert_eq!( - full.iter() - .map(|s| (s.row_range_start, s.row_range_end)) - .collect::>(), - vec![(0, 9), (10, 19), (20, 24)], - "first build must cover the full row range across shards" - ); -} - -/// Planner-level mid-coverage hole, mirroring btree's -/// `incremental_build_splits_gap_around_mid_coverage_indexed_hole`: with a -/// single shard cell (rows_per_shard large enough to hold all data) the grid -/// never splits, so the only split is the indexed hole itself. An indexed -/// range strictly inside the data coverage must carve the build into exactly -/// the two contiguous segments on either side of the hole -- both bounds -/// pinned, and neither segment may span or touch the hole. -#[test] -fn lumina_plan_splits_gap_around_mid_coverage_indexed_hole() { - // Data row-ids [0, 9]; one shard cell [0, 99] so the grid never splits. - let n = 9; - let hole_start = 4; - let hole_end = 6; - let shards = plan_with_indexed( - vec![manifest_entry(data_file("a", Some(0), n + 1))], - 100, - &[RowRange::new(hole_start, hole_end)], - ) - .unwrap(); - - let ranges = shards - .iter() - .map(|s| (s.row_range_start, s.row_range_end)) - .collect::>(); - // Exactly the two contiguous segments around the hole. - assert_eq!( - ranges, - vec![(0, hole_start - 1), (hole_end + 1, n)], - "mid-coverage hole must split into exactly the two segments around it" - ); - // Every emitted range is contiguous and none spans or touches the hole. - for (start, end) in &ranges { - assert!(end >= start, "range must be non-empty: [{start}, {end}]"); - assert!( - *end < hole_start || *start > hole_end, - "shard [{start}, {end}] must not overlap indexed hole [{hole_start}, {hole_end}]" - ); - } - // Together the shards cover exactly coverage - indexed. - let expected = exclude_row_ranges( - &[RowRange::new(0, n)], - &[RowRange::new(hole_start, hole_end)], - ) - .into_iter() - .map(|r| (r.from(), r.to())) - .collect::>(); - assert_eq!( - ranges, expected, - "shards must cover exactly coverage minus the indexed hole" - ); -} - -/// Planner-level incremental prefix. Strengthens -/// `lumina_incremental_build_indexes_only_new_rows`, which asserted only a -/// one-sided lower bound (`row_range_start >= n`): an indexed prefix [0, k] -/// must leave EXACTLY the suffix [k+1, N] on both bounds, split along the -/// shard grid, with nothing re-indexed inside the prefix. -#[test] -fn lumina_plan_incremental_prefix_leaves_suffix() { - // Data row-ids [0, 24], rows_per_shard = 10 -> cells [0,9],[10,19],[20,29]. - // Indexed prefix [0, 9] fully fills the first cell, so the build must be - // exactly [10, 19] and [20, 24] (the suffix split along the grid). - let n = 24; - let k = 9; // prefix [0, k] == the first full shard cell - let shards = plan_with_indexed( - vec![manifest_entry(data_file("a", Some(0), n + 1))], - 10, - &[RowRange::new(0, k)], - ) - .unwrap(); - - let ranges = shards - .iter() - .map(|s| (s.row_range_start, s.row_range_end)) - .collect::>(); - assert_eq!( - ranges, - vec![(k + 1, 19), (20, n)], - "indexed prefix must leave exactly the suffix, split along the shard grid" - ); - // Both bounds pinned (this is what the one-sided existing check omits). - assert_eq!(ranges.first().unwrap().0, k + 1, "suffix must start at k+1"); - assert_eq!(ranges.last().unwrap().1, n, "suffix must end at N"); - // Contiguous, and no shard reaches back into the indexed prefix. - for pair in ranges.windows(2) { - assert_eq!( - pair[1].0, - pair[0].1 + 1, - "ranges must be contiguous: {:?} then {:?}", - pair[0], - pair[1] - ); - } - for (start, end) in &ranges { - assert!( - *start > k, - "shard [{start}, {end}] must not re-index the prefix [0, {k}]" - ); - } -} diff --git a/crates/paimon/src/table/lumina_index_build_builder/validation.rs b/crates/paimon/src/table/lumina_index_build_builder/validation.rs deleted file mode 100644 index ec563571c..000000000 --- a/crates/paimon/src/table/lumina_index_build_builder/validation.rs +++ /dev/null @@ -1,135 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -//! Lumina table, field, option, and numeric validation. - -use crate::lumina::{LuminaVectorIndexOptions, LUMINA_DIMENSION_OPTION}; -use crate::spec::{CoreOptions, DataField, DataType}; -use crate::table::global_index_build_common::vector::{ - find_index_field as find_vector_index_field, - validate_vector_field as validate_common_vector_field, -}; -use crate::table::Table; -use crate::{Error, Result}; -use std::collections::HashMap; - -pub(super) fn validate_table_options(table: &Table, core_options: &CoreOptions) -> Result<()> { - if !table.schema().primary_keys().is_empty() { - return Err(Error::Unsupported { - message: "Lumina index build does not support primary-key tables".to_string(), - }); - } - if !core_options.row_tracking_enabled() { - return Err(Error::DataInvalid { - message: "Lumina index build requires 'row-tracking.enabled' = 'true'".to_string(), - source: None, - }); - } - if !core_options.data_evolution_enabled() { - return Err(Error::DataInvalid { - message: "Lumina index build requires 'data-evolution.enabled' = 'true'".to_string(), - source: None, - }); - } - if !core_options.global_index_enabled() { - return Err(Error::DataInvalid { - message: "Lumina index build requires 'global-index.enabled' = 'true'".to_string(), - source: None, - }); - } - if core_options.deletion_vectors_enabled() { - return Err(Error::Unsupported { - message: - "Lumina index build does not support tables with deletion-vectors.enabled=true" - .to_string(), - }); - } - Ok(()) -} - -pub(super) fn find_index_field<'a>(table: &'a Table, column: &str) -> Result<&'a DataField> { - find_vector_index_field(table, column) -} - -pub(super) fn validate_vector_field(field: &DataField) -> Result<()> { - validate_common_vector_field(field, "Lumina") -} - -pub(super) fn effective_lumina_options( - field: &DataField, - mut resolved: HashMap, -) -> Result> { - let DataType::Vector(vector) = field.data_type() else { - return Ok(resolved); - }; - let dimension = vector.length().to_string(); - match resolved.get(LUMINA_DIMENSION_OPTION) { - None => { - resolved.insert(LUMINA_DIMENSION_OPTION.to_string(), dimension); - } - Some(existing) if *existing == dimension => {} - Some(existing) => { - return Err(Error::ConfigInvalid { - message: format!( - "Vector column '{}' has dimension {} from its type, but '{}' is set to '{}'. \ - Remove the option or set it to {}.", - field.name(), - dimension, - LUMINA_DIMENSION_OPTION, - existing, - dimension - ), - }); - } - } - Ok(resolved) -} - -pub(super) fn resolve_lumina_options( - table_options: &HashMap, - user_options: &HashMap, -) -> Result> { - let mut options = table_options.clone(); - options.extend(user_options.clone()); - LuminaVectorIndexOptions::new(&options)?; - Ok(options) -} - -pub(super) fn checked_i64(value: u64, context: &str) -> Result { - i64::try_from(value).map_err(|_| Error::DataInvalid { - message: format!("{context}: {value}"), - source: None, - }) -} - -pub(super) fn checked_row_count(row_range_start: i64, row_range_end: i64) -> Result { - if row_range_end < row_range_start { - return Err(Error::DataInvalid { - message: format!("Invalid Lumina row range [{row_range_start}, {row_range_end}]"), - source: None, - }); - } - row_range_end - .checked_sub(row_range_start) - .and_then(|span| span.checked_add(1)) - .ok_or_else(|| Error::DataInvalid { - message: format!( - "Row count overflows for row range [{row_range_start}, {row_range_end}]" - ), - source: None, - }) -} diff --git a/crates/paimon/src/table/lumina_index_build_builder/writer.rs b/crates/paimon/src/table/lumina_index_build_builder/writer.rs deleted file mode 100644 index 8ad84596e..000000000 --- a/crates/paimon/src/table/lumina_index_build_builder/writer.rs +++ /dev/null @@ -1,184 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -//! Lumina index-file construction, upload, and cleanup. - -use super::planning::LuminaIndexShard; -use super::validation::{checked_i64, checked_row_count}; -use super::LuminaIndexBuildBuilder; -use crate::lumina::ffi::LuminaBuilder; -use crate::lumina::{LuminaIndexMeta, LUMINA_IDENTIFIER}; -use crate::spec::{GlobalIndexMeta, IndexFileMeta}; -use crate::table::global_index_build_common::copy_local_file_to_output; -use crate::table::{CommitMessage, TableCommit}; -use crate::{Error, Result}; -use std::path::PathBuf; - -impl LuminaIndexBuildBuilder<'_> { - pub(super) async fn build_index_file( - &self, - shard: &LuminaIndexShard, - vectors: &[f32], - dimension: i32, - index_field_id: i32, - index_meta: Vec, - ) -> Result { - let row_count = checked_row_count(shard.row_range_start, shard.row_range_end)?; - // The native Lumina builder counts rows in an i32; the manifest keeps the - // full width. - let native_row_count = i32::try_from(row_count).map_err(|_| Error::DataInvalid { - message: format!( - "Lumina shard row count {row_count} exceeds what the native builder accepts" - ), - source: None, - })?; - validate_vector_buffer(vectors, native_row_count, dimension)?; - let ids = (0..row_count as u64).collect::>(); - let native_options = LuminaIndexMeta::deserialize(&index_meta)?.options().clone(); - - let temp_path = temp_lumina_path(); - let temp_file = TempFileGuard::new(temp_path.clone()); - let temp_path_str = temp_path.to_string_lossy().to_string(); - let builder = LuminaBuilder::create(&native_options)?; - builder.pretrain(vectors, native_row_count, dimension)?; - builder.insert(vectors, &ids, native_row_count, dimension)?; - builder.dump(&temp_path_str)?; - - let file_name = format!("lumina-global-index-{}.index", uuid::Uuid::new_v4()); - let (index_path, external_path) = - crate::table::global_index_build_common::prepare_index_file_path( - self.table, &file_name, - ) - .await?; - let write_result: Result = async { - copy_local_file_to_output(&temp_path, self.table.file_io().new_output(&index_path)?) - .await?; - temp_file.cleanup(); - let status = self.table.file_io().get_status(&index_path).await?; - checked_i64( - status.size, - "Index file is too large for Rust IndexFileMeta", - ) - } - .await; - let file_size = match write_result { - Ok(file_size) => file_size, - Err(error) => { - let _ = self.table.file_io().delete_file(&index_path).await; - return Err(error); - } - }; - Ok(IndexFileMeta { - index_type: LUMINA_IDENTIFIER.to_string(), - file_name, - file_size, - row_count, - deletion_vectors_ranges: None, - external_path, - global_index_meta: Some(GlobalIndexMeta { - row_range_start: shard.row_range_start, - row_range_end: shard.row_range_end, - index_field_id, - extra_field_ids: None, - source_meta: Some( - crate::spec::DataEvolutionIndexSourceMeta::new(shard.snapshot_id)?.serialize(), - ), - index_meta: Some(index_meta), - }), - }) - } -} - -pub(super) async fn abort_on_build_error( - commit: &TableCommit, - messages: &[CommitMessage], - result: Result, -) -> Result { - match result { - Ok(value) => Ok(value), - Err(error) => { - // This build has not submitted or exposed any prepared shard. - let _ = commit.abort(messages).await; - Err(error) - } - } -} - -fn validate_vector_buffer(vectors: &[f32], row_count: i32, dimension: i32) -> Result<()> { - if row_count <= 0 { - return Err(Error::DataInvalid { - message: format!("Lumina shard row count must be positive, got: {row_count}"), - source: None, - }); - } - if dimension <= 0 { - return Err(Error::DataInvalid { - message: format!("Lumina vector dimension must be positive, got: {dimension}"), - source: None, - }); - } - let row_count = row_count as usize; - let dimension = dimension as usize; - let expected_len = row_count - .checked_mul(dimension) - .ok_or_else(|| Error::DataInvalid { - message: format!( - "Lumina vector buffer length overflows: row_count={row_count}, dimension={dimension}" - ), - source: None, - })?; - if vectors.len() != expected_len { - return Err(Error::DataInvalid { - message: format!( - "Lumina vector buffer length {} does not match row_count={} and dimension={}", - vectors.len(), - row_count, - dimension - ), - source: None, - }); - } - Ok(()) -} - -pub(super) fn temp_lumina_path() -> PathBuf { - std::env::temp_dir().join(format!("lumina-index-{}.index", uuid::Uuid::new_v4())) -} - -pub(super) struct TempFileGuard { - path: Option, -} - -impl TempFileGuard { - pub(super) fn new(path: PathBuf) -> Self { - Self { path: Some(path) } - } - - fn cleanup(mut self) { - if let Some(path) = self.path.take() { - let _ = std::fs::remove_file(path); - } - } -} - -impl Drop for TempFileGuard { - fn drop(&mut self) { - if let Some(path) = self.path.take() { - let _ = std::fs::remove_file(path); - } - } -} diff --git a/crates/paimon/src/table/mod.rs b/crates/paimon/src/table/mod.rs index 86b8f54de..a5eb5e2b0 100644 --- a/crates/paimon/src/table/mod.rs +++ b/crates/paimon/src/table/mod.rs @@ -96,7 +96,6 @@ mod inline_blob; mod instant; mod kv_file_reader; mod kv_file_writer; -mod lumina_index_build_builder; mod managed_blob_reader; mod managed_blob_reference; #[cfg(test)] diff --git a/crates/paimon/src/table/pk_vector_read.rs b/crates/paimon/src/table/pk_vector_read.rs index 37382da55..c6e3c0026 100644 --- a/crates/paimon/src/table/pk_vector_read.rs +++ b/crates/paimon/src/table/pk_vector_read.rs @@ -19,8 +19,6 @@ use crate::arrow::format::FilePredicates; use crate::arrow::residual::{evaluate_predicates_mask, widen_scan_fields}; -use crate::lumina::reader::LuminaVectorGlobalIndexReader; -use crate::lumina::LuminaIndexMeta; use crate::spec::{CoreOptions, DataField, Predicate}; use crate::table::bucket_filter::split_partition_and_data_predicates; use crate::table::data_file_reader::DataFileReader; @@ -40,10 +38,11 @@ use crate::table::source::DataSplit; use crate::table::vector_read::Read; use crate::table::vector_search_common::{ collect_ranked_rows, current_tokio_runtime_handle, log_vindex_range_io_stats, - reorder_and_strip_position, vindex_concurrency_limits, RankedRow, VectorIndexBackend, + reorder_and_strip_position, vindex_concurrency_limits, RankedRow, }; use crate::table::{ArrowRecordBatchStream, RowRange, Table}; use crate::vector_search::{GlobalIndexIOMeta, GlobalIndexResult, VectorSearch}; +use crate::vindex::is_vindex_index_type; use crate::vindex::pkvector::ann::{AnnSegmentSource, PkVectorAnnSearcher, VindexAnnSearcher}; use crate::vindex::pkvector::bucket::{BucketActiveFile, BucketAnnSegment, ExactFileSearchFuture}; use crate::vindex::pkvector::metric::VectorSearchMetric; @@ -53,7 +52,6 @@ use crate::vindex::reader::VindexVectorGlobalIndexReader; use arrow_array::{Array, Int64Array, RecordBatch}; use futures::{stream, TryStreamExt}; use std::collections::{HashMap, HashSet}; -use std::io::Cursor; use std::sync::Arc; pub(super) struct PkVectorRead { @@ -269,27 +267,22 @@ async fn search_pk_raw_candidates_batch_with_plan( let skip_exact_fallback = params.skip_exact_fallback; let indexed_limit = params.indexed_limit; - // Resolve the vector index backend from the single configured index type. - // Java enforces one index type per PK table and Rust filters segments to it, - // so one backend serves every segment. Computed after the empty-plan return so - // an empty table never errors on an unrecognized type. - let backend = VectorIndexBackend::from_index_type(&index_type).ok_or_else(|| { - crate::Error::DataInvalid { + // All segments use the table's configured vindex type. An empty plan returns + // above before opening readers or resolving concurrency limits. + if !is_vindex_index_type(&index_type) { + return Err(crate::Error::DataInvalid { message: format!("unsupported PK vector index backend/type: '{index_type}'"), source: None, - } - })?; - let (batch_index_parallelism, range_read_concurrency) = match backend { - VectorIndexBackend::Vindex => vindex_concurrency_limits( - core, - plan.splits - .iter() - .map(|split| split.ann_segments.len()) - .sum(), - concurrency, - )?, - VectorIndexBackend::Lumina => (1, 0), - }; + }); + } + let (batch_index_parallelism, range_read_concurrency) = vindex_concurrency_limits( + core, + plan.splits + .iter() + .map(|split| split.ann_segments.len()) + .sum(), + concurrency, + )?; // Production data-file reader, mirroring `table_read.rs::new_data_file_reader` // but projecting only the vector column with no predicates. @@ -305,8 +298,8 @@ async fn search_pk_raw_candidates_batch_with_plan( .with_table_options(table.schema().options().clone()); // Real ANN scorer + loader. Each segment source is opened lazily inside its - // bucket leaf and dropped after scoring. Lumina keeps its buffered-byte path; - // vindex remains range-backed and reads only metadata and probed lists. + // bucket leaf and dropped after scoring. Range reads load only metadata + // and the index data needed for the search. let options = { let mut o = table.schema().options().clone(); o.extend(query_options.clone()); @@ -316,52 +309,36 @@ async fn search_pk_raw_candidates_batch_with_plan( let field_name = pk_col.to_string(); let loader_io = table.file_io().clone(); - let loader_range_read_limiter = match backend { - VectorIndexBackend::Vindex => Some(RangeReadLimiter::new(range_read_concurrency)), - VectorIndexBackend::Lumina => None, - }; - let loader: crate::vindex::pkvector::ann::SourceSegmentLoader = Box::new( - move |segment: &BucketAnnSegment| { + let loader_range_read_limiter = RangeReadLimiter::new(range_read_concurrency); + let loader: crate::vindex::pkvector::ann::SourceSegmentLoader = + Box::new(move |segment: &BucketAnnSegment| { let io = loader_io.clone(); let range_read_limiter = loader_range_read_limiter.clone(); let path = segment.path.clone(); let file_size = segment.file_size; Box::pin(async move { let input = io.new_input(&path)?; - match backend { - VectorIndexBackend::Lumina => input - .read() + let file_reader = + input + .reader() .await - .map(AnnSegmentSource::Buffered) .map_err(|error| crate::Error::DataInvalid { - message: format!("failed to read ANN index file '{path}': {error}"), - source: None, - }), - VectorIndexBackend::Vindex => { - let file_reader = - input - .reader() - .await - .map_err(|error| crate::Error::DataInvalid { - message: format!( - "failed to open ANN index file '{path}' for range reads: {error}" - ), - source: None, - })?; - Ok(AnnSegmentSource::Vindex( - VindexFileReader::new_with_limiter( - Arc::new(file_reader), - current_tokio_runtime_handle()?, - range_read_limiter.expect("Vindex range-read limiter"), - file_size, - path, + message: format!( + "failed to open ANN index file '{path}' for range reads: {error}" ), - )) - } - } + source: None, + })?; + Ok(AnnSegmentSource::Vindex( + VindexFileReader::new_with_limiter( + Arc::new(file_reader), + current_tokio_runtime_handle()?, + range_read_limiter, + file_size, + path, + ), + )) }) - }, - ); + }); let scorer: crate::vindex::pkvector::ann::SourceBatchScorer = Box::new( move |segment: &BucketAnnSegment, source: AnnSegmentSource, searches: &[VectorSearch]| { @@ -370,15 +347,8 @@ async fn search_pk_raw_candidates_batch_with_plan( segment.file_size, segment.index_meta.clone(), ); - match (backend, source) { - (VectorIndexBackend::Lumina, AnnSegmentSource::Buffered(data)) => { - let lumina_metric = - LuminaIndexMeta::deserialize(&segment.index_meta)?.metric()?; - verify_segment_metric(metric, VectorSearchMetric::from_lumina(lumina_metric))?; - let mut reader = LuminaVectorGlobalIndexReader::new(io_meta, options.clone()); - reader.visit_batch_vector_search(searches, |_| Ok(Cursor::new(data))) - } - (VectorIndexBackend::Vindex, AnnSegmentSource::Vindex(source)) => { + match source { + AnnSegmentSource::Vindex(source) => { let range_io_stats = source.range_io_stats(); let mut reader = VindexVectorGlobalIndexReader::new(io_meta, options.clone()) .with_batch_index_parallelism(batch_index_parallelism); @@ -397,16 +367,13 @@ async fn search_pk_raw_candidates_batch_with_plan( } Ok(results) } - (VectorIndexBackend::Lumina, AnnSegmentSource::Vindex(_)) - | (VectorIndexBackend::Vindex, AnnSegmentSource::Buffered(_)) => { - Err(crate::Error::DataInvalid { - message: format!( - "ANN segment '{}' was loaded with the wrong backend source", - segment.path - ), - source: None, - }) - } + AnnSegmentSource::Buffered(_) => Err(crate::Error::DataInvalid { + message: format!( + "ANN segment '{}' was loaded with the wrong backend source", + segment.path + ), + source: None, + }), } }, ); diff --git a/crates/paimon/src/table/pk_vector_read/tests.rs b/crates/paimon/src/table/pk_vector_read/tests.rs index 10fa3ee3a..37af1c2b1 100644 --- a/crates/paimon/src/table/pk_vector_read/tests.rs +++ b/crates/paimon/src/table/pk_vector_read/tests.rs @@ -18,7 +18,6 @@ use super::*; use crate::catalog::Identifier; use crate::io::{FileIO, FileIOBuilder}; -use crate::lumina::LuminaIndexMeta; use crate::spec::stats::BinaryTableStats; use crate::spec::{ BinaryRow, DataField, DataFileMeta, DataType, Datum, FloatType, IntType, Predicate, @@ -30,7 +29,6 @@ use crate::table::source::DataSplitBuilder; use crate::table::vector_scan::Scan; use crate::table::vector_search_test_utils::{build_vindex_segment_bytes, pk_vector_table}; use crate::table::{Table, TableCommit, TableWrite}; -use crate::vindex::pkvector::bucket::BucketAnnSegment; use crate::vindex::pkvector::metric::VectorSearchMetric; use crate::vindex::IVF_FLAT_IDENTIFIER; use arrow_array::builder::{FixedSizeListBuilder, Float32Builder}; @@ -99,22 +97,6 @@ fn pk_data_file(name: &str, row_count: i64, first_row_id: Option) -> DataFi } } -fn pk_search_split(bucket: i32, files: Vec) -> PkVectorSearchSplit { - PkVectorSearchSplit { - data_split: DataSplitBuilder::new() - .with_snapshot(1) - .with_partition(BinaryRow::new(0)) - .with_bucket(bucket) - .with_bucket_path(format!("memory:/t/bucket-{bucket}")) - .with_total_buckets(1) - .with_data_files(files) - .build() - .unwrap(), - ann_segments: Vec::new(), - active_files: Vec::new(), - } -} - fn pk_candidate( split_index: usize, bucket: i32, @@ -287,26 +269,6 @@ async fn vector_rerank_fixture( (reader, split, vector_field) } -fn pk_split_with_lumina_segment(path: &str, metric: &str) -> PkVectorSearchSplit { - let mut split = pk_search_split(0, vec![pk_data_file("file-a", 3, Some(0))]); - let source_meta = crate::spec::PrimaryKeyIndexSourceMeta::new( - 1, - vec![crate::spec::PrimaryKeyIndexSourceFile::new("file-a".to_string(), 3).unwrap()], - ) - .unwrap(); - let mut segment = BucketAnnSegment::for_test(source_meta); - segment.path = path.to_string(); - // Lumina stores its metric in the serialized index metadata blob, not in - // the segment file bytes. `deserialize` requires both keys present. - let meta = crate::lumina::LuminaIndexMeta::new(HashMap::from([ - ("index.dimension".to_string(), "2".to_string()), - ("distance.metric".to_string(), metric.to_string()), - ])); - segment.index_meta = meta.serialize().unwrap(); - split.ann_segments = vec![segment]; - split -} - /// One Java `DataOutput#writeUTF` value (u16-BE length + modified UTF-8), used /// to assemble the `PrimaryKeyIndexSourceMeta` frame below. fn java_write_utf(s: &str) -> Vec { @@ -947,47 +909,6 @@ async fn rerank_reads_only_candidate_positions_not_whole_column() { assert_eq!(out[1].distance, 4.0); } -#[test] -fn verify_segment_metric_accepts_matching_lumina_metric() { - // Lumina segment metadata says cosine; configured cosine => Ok. No segment - // file bytes are needed on the Lumina path. - let split = pk_split_with_lumina_segment("seg-lumina", "cosine"); - let segment = &split.ann_segments[0]; - let lumina_metric = LuminaIndexMeta::deserialize(&segment.index_meta) - .unwrap() - .metric() - .unwrap(); - verify_segment_metric( - VectorSearchMetric::Cosine, - VectorSearchMetric::from_lumina(lumina_metric), - ) - .expect("matching lumina metric must pass"); -} - -#[test] -fn verify_segment_metric_rejects_mismatched_lumina_metric() { - // Lumina segment metadata says l2; configured inner_product => fail loud, - // naming both metrics. - let split = pk_split_with_lumina_segment("seg-lumina", "l2"); - let segment = &split.ann_segments[0]; - let lumina_metric = LuminaIndexMeta::deserialize(&segment.index_meta) - .unwrap() - .metric() - .unwrap(); - let err = verify_segment_metric( - VectorSearchMetric::InnerProduct, - VectorSearchMetric::from_lumina(lumina_metric), - ) - .expect_err("mismatched lumina metric must fail loud"); - assert!( - matches!(err, crate::Error::DataInvalid { ref message, .. } - if message.contains("does not match configured metric") - && message.contains("l2") - && message.contains("inner_product")), - "unexpected error: {err:?}" - ); -} - #[test] fn verify_segment_metric_accepts_matching_vindex_metric() { // Real IVF segment trained with L2; configured metric L2 => Ok. diff --git a/crates/paimon/src/table/pk_vector_scan.rs b/crates/paimon/src/table/pk_vector_scan.rs index 7d40fa320..0e9c22c00 100644 --- a/crates/paimon/src/table/pk_vector_scan.rs +++ b/crates/paimon/src/table/pk_vector_scan.rs @@ -410,9 +410,6 @@ impl Scan for PkVectorScan { file_name: entry.index_file.file_name.clone(), external_path: entry.index_file.external_path.clone(), file_size, - // The Lumina reader consumes this as its serialized index - // metadata; the vindex reader ignores it and loads metadata - // from the segment file bytes. Absent value defaults to empty. index_meta: gim.index_meta.clone().unwrap_or_default(), }, )); @@ -1747,7 +1744,7 @@ mod tests { // The chain the two tests above only cover in halves: the committed // Java-planned fixture, through planning, into the function that decides // whether the ANN backend is handed an `include_row_ids` filter. `None` here - // is what makes Lumina call `search` rather than `search_with_filter`. + // lets the index search without a row-id filter. // Result equality against the manifest route cannot see this difference -- // both return the same rows for this fixture. let split = BucketVectorSearchSplit::deserialize(BUCKET_SPLIT_NO_PREFILTER).unwrap(); diff --git a/crates/paimon/src/table/pk_vector_search_params.rs b/crates/paimon/src/table/pk_vector_search_params.rs index fbd751db9..9058853aa 100644 --- a/crates/paimon/src/table/pk_vector_search_params.rs +++ b/crates/paimon/src/table/pk_vector_search_params.rs @@ -17,7 +17,6 @@ //! Resolve and validate PK vector-search parameters before scan planning. -use crate::lumina::{is_lumina_index_type, LuminaVectorIndexOptions}; use crate::spec::{CoreOptions, DataField, DataType, GlobalIndexSearchMode, Predicate}; use crate::table::bucket_filter::split_partition_and_data_predicates; use crate::table::vector_search_common::{configured_refine_factor, indexed_search_limit}; @@ -176,55 +175,42 @@ fn pk_vector_query_dimension( Ok(Some(vector_type.length() as usize)) } DataType::Array(array_type) if matches!(array_type.element_type(), DataType::Float(_)) => { - // Resolve the dimension per the configured backend. An `ARRAY` - // column carries no dimension in its type, so it comes from options — - // but the option shape differs by backend. Lumina is not a vindex - // index type, so routing it through `VindexVectorIndexOptions` would - // reject it as unsupported before planning (even on an empty table). - if is_lumina_index_type(index_type) { - // Lumina reads `lumina.index.dimension` (default 128) from the - // merged table+query options, matching `resolve_lumina_options`. - let mut merged = table_options.clone(); - merged.extend(query_options.clone()); - let dimension = LuminaVectorIndexOptions::new(&merged)?.dimension; - Ok(Some(dimension as usize)) - } else { - let mut dimension_options = HashMap::new(); - for key in [ - "dimension".to_string(), - format!("{index_type}.dimension"), - format!("{index_type}.index.dimension"), - format!("fields.{}.dimension", vector_field.name()), - format!("fields.{}.index.dimension", vector_field.name()), - ] { - if let Some(value) = query_options.get(&key) { - let key = if key == "dimension" { - format!("{index_type}.dimension") - } else { - key - }; - dimension_options.insert(key, value.clone()); - } + // ARRAY carries its dimension in the vindex options. + let mut dimension_options = HashMap::new(); + for key in [ + "dimension".to_string(), + format!("{index_type}.dimension"), + format!("{index_type}.index.dimension"), + format!("fields.{}.dimension", vector_field.name()), + format!("fields.{}.index.dimension", vector_field.name()), + ] { + if let Some(value) = query_options.get(&key) { + let key = if key == "dimension" { + format!("{index_type}.dimension") + } else { + key + }; + dimension_options.insert(key, value.clone()); } - let mut table_dimension_options = table_options.clone(); - if let Some(value) = table_options.get("dimension") { - if !table_options.contains_key(&format!("{index_type}.dimension")) - && !table_options.contains_key(&format!("{index_type}.index.dimension")) - { - table_dimension_options - .insert(format!("{index_type}.dimension"), value.clone()); - } + } + let mut table_dimension_options = table_options.clone(); + if let Some(value) = table_options.get("dimension") { + if !table_options.contains_key(&format!("{index_type}.dimension")) + && !table_options.contains_key(&format!("{index_type}.index.dimension")) + { + table_dimension_options + .insert(format!("{index_type}.dimension"), value.clone()); } - Ok(Some( - VindexVectorIndexOptions::new( - &table_dimension_options, - &dimension_options, - index_type, - vector_field, - )? - .dimension(), - )) } + Ok(Some( + VindexVectorIndexOptions::new( + &table_dimension_options, + &dimension_options, + index_type, + vector_field, + )? + .dimension(), + )) } _ => Ok(None), } diff --git a/crates/paimon/src/table/search_scan_split.rs b/crates/paimon/src/table/search_scan_split.rs index 438a6e241..200f97e10 100644 --- a/crates/paimon/src/table/search_scan_split.rs +++ b/crates/paimon/src/table/search_scan_split.rs @@ -21,11 +21,12 @@ use super::bucket_filter::split_partition_and_data_predicates; use super::global_index_scanner::unindexed_ranges_for_global_index_entries; use super::global_index_types::normalize_queryable_global_index_type; use super::row_id_predicate::intersect_sorted_ranges; -use super::vector_search_common::{configured_vector_index_type, VectorIndexBackend}; +use super::vector_search_common::configured_vector_index_type; use super::{find_field_id_by_name, merge_row_ranges, DataSplit, RowRange, Table}; use crate::spec::{ CoreOptions, FileKind, GlobalIndexSearchMode, IndexFileMeta, IndexManifestEntry, Predicate, }; +use crate::vindex::is_vindex_index_type; use serde::{Deserialize, Serialize}; use std::collections::{BTreeMap, HashMap, HashSet}; @@ -80,7 +81,7 @@ pub(super) async fn global_search_splits( && if full_text { file.index_type == "full-text" } else { - VectorIndexBackend::from_index_type(&file.index_type).is_some() + is_vindex_index_type(&file.index_type) } }; let selected = entries @@ -450,7 +451,7 @@ mod tests { fn raw_source_scalar_metadata_cannot_add_search_payloads() { // A search payload hidden in scalar metadata would otherwise be searched // over [0, 2], extending the explicitly selected raw source [1, 1]. - for index_type in ["ivf-flat", "lumina", "lumina-vector-ann", "full-text"] { + for index_type in ["ivf-flat", "diskann", "full-text"] { let source = raw_source_with_scalar(index_type); let error = VectorScanPlan::new(Some(snapshot()), vec![source.clone()]).unwrap_err(); assert!(error.to_string().contains("unsupported index type")); diff --git a/crates/paimon/src/table/table_commit.rs b/crates/paimon/src/table/table_commit.rs index 72e7ec2ba..1df59e0fc 100644 --- a/crates/paimon/src/table/table_commit.rs +++ b/crates/paimon/src/table/table_commit.rs @@ -4097,7 +4097,7 @@ mod tests { row_range_end: i64, ) -> IndexFileMeta { IndexFileMeta { - index_type: "lumina".to_string(), + index_type: "diskann".to_string(), file_name: name.to_string(), file_size: 128, row_count: (row_range_end - row_range_start + 1), @@ -4965,7 +4965,7 @@ mod tests { .unwrap(); let mut message = CommitMessage::new(vec![], 0, vec![]); - message.new_index_files = vec![test_global_index_file("lumina-0.index", 0, 0, 9)]; + message.new_index_files = vec![test_global_index_file("diskann-0.index", 0, 0, 9)]; commit .commit_if_latest_snapshot(vec![message], 1) .await @@ -4985,7 +4985,7 @@ mod tests { .await .unwrap(); assert_eq!(index_entries.len(), 1); - assert_eq!(index_entries[0].index_file.file_name, "lumina-0.index"); + assert_eq!(index_entries[0].index_file.file_name, "diskann-0.index"); } #[tokio::test] @@ -5002,7 +5002,7 @@ mod tests { .await .unwrap(); - let index_path = format!("{table_path}/index/lumina-0.index"); + let index_path = format!("{table_path}/index/diskann-0.index"); file_io .mkdirs(&format!("{table_path}/index/")) .await @@ -5014,7 +5014,7 @@ mod tests { .await .unwrap(); let mut message = CommitMessage::new(vec![], 0, vec![]); - message.new_index_files = vec![test_global_index_file("lumina-0.index", 0, 0, 9)]; + message.new_index_files = vec![test_global_index_file("diskann-0.index", 0, 0, 9)]; let result = commit.commit_if_latest_snapshot(vec![message], 0).await; assert!(result.is_err()); @@ -5085,11 +5085,11 @@ mod tests { let commit = setup_commit(&file_io, table_path); let mut first = CommitMessage::new(vec![], 0, vec![]); - first.new_index_files = vec![test_global_index_file("lumina-0.index", 0, 0, 9)]; + first.new_index_files = vec![test_global_index_file("diskann-0.index", 0, 0, 9)]; commit.commit(vec![first]).await.unwrap(); let mut second = CommitMessage::new(vec![], 0, vec![]); - second.new_index_files = vec![test_global_index_file("lumina-1.index", 0, 5, 14)]; + second.new_index_files = vec![test_global_index_file("diskann-1.index", 0, 5, 14)]; let result = commit.commit(vec![second]).await; assert!(result.is_err()); @@ -5109,8 +5109,8 @@ mod tests { let commit = setup_commit(&file_io, table_path); let mut message = CommitMessage::new(vec![], 0, vec![]); message.new_index_files = vec![ - test_global_index_file("lumina-0.index", 0, 0, 9), - test_global_index_file("lumina-1.index", 0, 5, 14), + test_global_index_file("diskann-0.index", 0, 0, 9), + test_global_index_file("diskann-1.index", 0, 5, 14), ]; let result = commit.commit(vec![message]).await; @@ -5130,11 +5130,11 @@ mod tests { let commit = setup_commit(&file_io, table_path); let mut first = CommitMessage::new(vec![], 0, vec![]); - first.new_index_files = vec![test_global_index_file("lumina-0.index", 0, 0, 9)]; + first.new_index_files = vec![test_global_index_file("diskann-0.index", 0, 0, 9)]; commit.commit(vec![first]).await.unwrap(); let mut second = CommitMessage::new(vec![], 0, vec![]); - second.new_index_files = vec![test_global_index_file("lumina-1.index", 0, 10, 19)]; + second.new_index_files = vec![test_global_index_file("diskann-1.index", 0, 10, 19)]; commit.commit(vec![second]).await.unwrap(); let snap_manager = SnapshotManager::new(file_io.clone(), table_path.to_string()); @@ -5160,12 +5160,12 @@ mod tests { #[test] fn test_global_index_overlap_allows_different_index_type() { // Same field id (0) and overlapping row ranges, but the retained entry is - // a `lumina` index and the added one is `btree`: distinct identities that + // a `diskann` index and the added one is `btree`: distinct identities that // must coexist rather than trip the overlap guard. - let lumina = test_global_index_file("lumina-0.index", 0, 0, 9); + let diskann = test_global_index_file("diskann-0.index", 0, 0, 9); let mut btree = test_global_index_file("btree-0.index", 0, 5, 14); btree.index_type = "btree".to_string(); - let retained = vec![add_index_entry(lumina)]; + let retained = vec![add_index_entry(diskann)]; let added = vec![add_index_entry(btree)]; TableCommit::validate_global_index_overlap(&retained, &added) .expect("different index types on the same field must coexist"); @@ -5176,14 +5176,14 @@ mod tests { // Same index type and field id, overlapping ranges, but different // `extra_field_ids` -> distinct identities, must coexist. let retained = vec![add_index_entry(test_global_index_file_with_extra_fields( - "lumina-0.index", + "diskann-0.index", 0, vec![1], 0, 9, ))]; let added = vec![add_index_entry(test_global_index_file_with_extra_fields( - "lumina-1.index", + "diskann-1.index", 0, vec![2], 5, @@ -5198,13 +5198,13 @@ mod tests { // Identical identity (same type + field + extra) with overlapping ranges // must still be rejected. let retained = vec![add_index_entry(test_global_index_file( - "lumina-0.index", + "diskann-0.index", 0, 0, 9, ))]; let added = vec![add_index_entry(test_global_index_file( - "lumina-1.index", + "diskann-1.index", 0, 5, 14, @@ -5221,12 +5221,12 @@ mod tests { #[test] fn test_added_global_index_overlap_allows_different_index_type() { // Two entries added within the same commit: same field id (0) and - // overlapping row ranges, but distinct index types (`lumina` vs `btree`). + // overlapping row ranges, but distinct index types (`diskann` vs `btree`). // These are distinct global-index identities that must coexist. - let lumina = test_global_index_file("lumina-0.index", 0, 0, 9); + let diskann = test_global_index_file("diskann-0.index", 0, 0, 9); let mut btree = test_global_index_file("btree-0.index", 0, 5, 14); btree.index_type = "btree".to_string(); - let added = vec![add_index_entry(lumina), add_index_entry(btree)]; + let added = vec![add_index_entry(diskann), add_index_entry(btree)]; TableCommit::validate_added_global_index_overlap(&added) .expect("different index types on the same field must coexist within a commit"); } @@ -5238,14 +5238,14 @@ mod tests { // identities, must coexist. let added = vec![ add_index_entry(test_global_index_file_with_extra_fields( - "lumina-0.index", + "diskann-0.index", 0, vec![1], 0, 9, )), add_index_entry(test_global_index_file_with_extra_fields( - "lumina-1.index", + "diskann-1.index", 0, vec![2], 5, @@ -5261,8 +5261,8 @@ mod tests { // Two entries added within the same commit with identical identity // (same type + field + extra) and overlapping ranges must be rejected. let added = vec![ - add_index_entry(test_global_index_file("lumina-0.index", 0, 0, 9)), - add_index_entry(test_global_index_file("lumina-1.index", 0, 5, 14)), + add_index_entry(test_global_index_file("diskann-0.index", 0, 0, 9)), + add_index_entry(test_global_index_file("diskann-1.index", 0, 5, 14)), ]; let result = TableCommit::validate_added_global_index_overlap(&added); let err_msg = result.expect_err("same-identity overlap must be rejected"); @@ -5280,7 +5280,7 @@ mod tests { setup_dirs(&file_io, table_path).await; let commit = setup_commit(&file_io, table_path); - let index_file = test_global_index_file("lumina-0.index", 0, 0, 9); + let index_file = test_global_index_file("diskann-0.index", 0, 0, 9); let mut first = CommitMessage::new(vec![], 0, vec![]); first.new_index_files = vec![index_file.clone()]; commit.commit(vec![first]).await.unwrap(); @@ -5306,7 +5306,7 @@ mod tests { let commit = setup_commit(&file_io, table_path); let mut first = CommitMessage::new(vec![], 0, vec![]); - first.new_index_files = vec![test_global_index_file("lumina-0.index", 0, 0, 9)]; + first.new_index_files = vec![test_global_index_file("diskann-0.index", 0, 0, 9)]; commit.commit(vec![first]).await.unwrap(); commit @@ -5328,7 +5328,7 @@ mod tests { .await .unwrap(); assert_eq!(index_entries.len(), 1); - assert_eq!(index_entries[0].index_file.file_name, "lumina-0.index"); + assert_eq!(index_entries[0].index_file.file_name, "diskann-0.index"); } #[tokio::test] @@ -5491,7 +5491,7 @@ mod tests { let commit = setup_commit(&file_io, table_path); let mut first = CommitMessage::new(vec![], 0, vec![]); - first.new_index_files = vec![test_global_index_file("lumina-0.index", 0, 0, 9)]; + first.new_index_files = vec![test_global_index_file("diskann-0.index", 0, 0, 9)]; commit.commit(vec![first]).await.unwrap(); let mut data_file = test_data_file("data-update-id.parquet", 10); @@ -5528,7 +5528,7 @@ mod tests { let commit = TableCommit::new(table, "test-user".to_string()); let mut first = CommitMessage::new(vec![], 0, vec![]); - first.new_index_files = vec![test_global_index_file("lumina-0.index", 0, 0, 9)]; + first.new_index_files = vec![test_global_index_file("diskann-0.index", 0, 0, 9)]; commit.commit(vec![first]).await.unwrap(); let mut data_file = test_data_file("data-update-id.parquet", 10); @@ -5554,7 +5554,7 @@ mod tests { let commit = setup_commit(&file_io, table_path); let mut first = CommitMessage::new(vec![], 0, vec![]); first.new_index_files = vec![test_global_index_file_with_extra_fields( - "lumina-id-name.index", + "diskann-id-name.index", 0, vec![1], 0, @@ -5597,7 +5597,7 @@ mod tests { let mut first = CommitMessage::new(vec![], 0, vec![]); first.new_index_files = vec![test_global_index_file_with_extra_fields( - "lumina-id-name.index", + "diskann-id-name.index", 0, vec![1], 0, @@ -5626,7 +5626,7 @@ mod tests { let commit = setup_commit(&file_io, table_path); let mut first = CommitMessage::new(vec![], 0, vec![]); - first.new_index_files = vec![test_global_index_file("lumina-0.index", 0, 0, 9)]; + first.new_index_files = vec![test_global_index_file("diskann-0.index", 0, 0, 9)]; commit.commit(vec![first]).await.unwrap(); let mut data_file = test_data_file("data-update-name.parquet", 10); diff --git a/crates/paimon/src/table/table_write.rs b/crates/paimon/src/table/table_write.rs index 9a18df92d..d284e7d32 100644 --- a/crates/paimon/src/table/table_write.rs +++ b/crates/paimon/src/table/table_write.rs @@ -6201,7 +6201,7 @@ pub(in crate::table) mod tests { append(&table, 1).await.unwrap(); let mut message = crate::table::CommitMessage::new(vec![], 0, vec![]); message.new_index_files.push(crate::spec::IndexFileMeta { - index_type: "lumina".into(), + index_type: "diskann".into(), file_name: "audit-index".into(), file_size: 1, row_count: 1, diff --git a/crates/paimon/src/table/vector_read.rs b/crates/paimon/src/table/vector_read.rs index ebc7a64e7..91a1a7a72 100644 --- a/crates/paimon/src/table/vector_read.rs +++ b/crates/paimon/src/table/vector_read.rs @@ -140,10 +140,7 @@ impl BatchVectorRead { &self.context.column, ); if index_files.iter().any(|file| { - super::vector_search_common::VectorIndexBackend::from_index_type( - &file.index_type, - ) - .is_none() + !crate::vindex::is_vindex_index_type(&file.index_type) || file .global_index_meta .as_ref() diff --git a/crates/paimon/src/table/vector_search_builder/tests.rs b/crates/paimon/src/table/vector_search_builder/tests.rs index 40b27adef..756b7894b 100644 --- a/crates/paimon/src/table/vector_search_builder/tests.rs +++ b/crates/paimon/src/table/vector_search_builder/tests.rs @@ -577,22 +577,16 @@ async fn result_read_empty_plan_reserved_projection_fails_loud() { } #[tokio::test] -async fn result_read_empty_plan_lumina_array_float_is_admitted() { - // A Lumina PK-vector `ARRAY` column is a valid configuration, but - // batch query dimension validation routed every `ARRAY` column - // through the vindex resolver, which rejects `lumina` as an unsupported - // index type before planning — failing even an empty table. The - // dimension must be resolved per the configured backend, so a - // well-formed Lumina query is admitted and (with no snapshot) yields an - // empty stream rather than an "Unsupported vindex index type" error. +async fn result_read_empty_plan_diskann_array_float_is_admitted() { + // An empty PK-vector table resolves ARRAY dimensions from DiskANN options. let table = pk_vector_table(&[ ("pk-vector.index.columns", "embedding"), ( "fields.embedding.pk-vector.index.type", - crate::lumina::LUMINA_IDENTIFIER, + crate::vindex::DISKANN_IDENTIFIER, ), ("fields.embedding.pk-vector.distance.metric", "l2"), - ("lumina.index.dimension", "4"), + ("diskann.dimension", "4"), ]); let mut stream = async { table @@ -607,7 +601,7 @@ async fn result_read_empty_plan_lumina_array_float_is_admitted() { .await } .await - .expect("Lumina ARRAY query must be admitted, not rejected as unsupported vindex"); + .expect("DiskANN ARRAY query must be admitted"); assert!(stream.try_next().await.unwrap().is_none()); } diff --git a/crates/paimon/src/table/vector_search_common.rs b/crates/paimon/src/table/vector_search_common.rs index 9fd80e606..93eaea74c 100644 --- a/crates/paimon/src/table/vector_search_common.rs +++ b/crates/paimon/src/table/vector_search_common.rs @@ -17,7 +17,6 @@ //! Shared vector-search option validation, index I/O helpers, and result ordering. -use crate::lumina::is_lumina_index_type; use crate::spec::{ CoreOptions, DataField, FileKind, IndexManifestEntry, Predicate, ROW_ID_FIELD_NAME, }; @@ -39,31 +38,6 @@ pub(super) fn add_filter(current: &mut Option, filter: Predicate) { }); } -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -pub(super) enum VectorIndexBackend { - Lumina, - Vindex, -} - -impl VectorIndexBackend { - pub(super) fn from_index_type(index_type: &str) -> Option { - if is_lumina_index_type(index_type) { - Some(Self::Lumina) - } else if is_vindex_index_type(index_type) { - Some(Self::Vindex) - } else { - None - } - } - - pub(super) fn error_name(self) -> &'static str { - match self { - Self::Lumina => "Lumina", - Self::Vindex => "vindex", - } - } -} - /// A search column selects one persisted vector format, as in Java's scan. /// Different columns, removed entries and scalar indexes do not participate. pub(super) fn validate_global_vector_index_types( @@ -75,7 +49,7 @@ pub(super) fn validate_global_vector_index_types( for entry in entries { let file = &entry.index_file; if entry.kind != FileKind::Add - || VectorIndexBackend::from_index_type(&file.index_type).is_none() + || !is_vindex_index_type(&file.index_type) || !file .global_index_meta .as_ref() diff --git a/crates/paimon/src/table/vindex_index_build_builder/tests.rs b/crates/paimon/src/table/vindex_index_build_builder/tests.rs index cba9244de..f0b42111b 100644 --- a/crates/paimon/src/table/vindex_index_build_builder/tests.rs +++ b/crates/paimon/src/table/vindex_index_build_builder/tests.rs @@ -748,7 +748,7 @@ async fn vector_build_with_deletion_vectors_preserves_physical_coverage() { /// Commit a synthetic vindex `IndexFileMeta` covering `[start, end]` for /// `field_id` directly into the index manifest, without invoking the native -/// builder. Mirrors the Lumina/btree tests so the incremental gap logic can +/// builder. Mirrors the BTree tests so the incremental gap logic can /// be exercised without a trained vector index. Writes the same `index_type` /// (`ivf-flat`) the builder-under-test uses, so the gap helper matches it. async fn commit_synthetic_vindex_index(table: &Table, field_id: i32, start: i64, end: i64) { @@ -1333,7 +1333,7 @@ async fn vindex_build_cleans_written_shards_when_later_shard_fails() { .is_empty()); } -/// A field that already carries a DIFFERENT index type (`lumina`) over an +/// A field that already carries a DIFFERENT index type (`diskann`) over an /// overlapping row range must not block a vindex (`ivf-flat`) build on the /// same field: the two indexes have distinct identities and coexist. Before /// the full-identity fix, the overlap guard keyed only on field id + range @@ -1352,14 +1352,14 @@ async fn vindex_build_coexists_with_different_index_type_on_same_field() { ) .await; - // Pre-existing `lumina` index covering the full data range on the SAME + // Pre-existing `diskann` index covering the full data range on the SAME // field the vindex build will target. let coverage = data_row_id_coverage(&table).await; assert_eq!(coverage.len(), 1, "data must be one contiguous range"); let field_id = find_index_field(&table, "embedding").unwrap().id(); - let lumina = IndexFileMeta { - index_type: "lumina".to_string(), - file_name: "lumina-synthetic-0.index".to_string(), + let diskann = IndexFileMeta { + index_type: "diskann".to_string(), + file_name: "diskann-synthetic-0.index".to_string(), file_size: 1, row_count: (coverage[0].to() - coverage[0].from() + 1) as i64, deletion_vectors_ranges: None, @@ -1374,7 +1374,7 @@ async fn vindex_build_coexists_with_different_index_type_on_same_field() { }), }; let mut message = CommitMessage::new(BinaryRow::new(0).to_serialized_bytes(), 0, vec![]); - message.new_index_files = vec![lumina]; + message.new_index_files = vec![diskann]; TableCommit::new(table.clone(), "test-user".to_string()) .commit(vec![message]) .await @@ -1427,7 +1427,7 @@ fn vindex_first_build_indexes_full_coverage() { ); } -/// Planner-level mid-coverage hole, mirroring btree/lumina: with a single +/// Planner-level mid-coverage hole, mirroring BTree: with a single /// shard cell (rows_per_shard large enough to hold all data) the grid never /// splits, so the only split is the indexed hole itself. An indexed range /// strictly inside the data coverage must carve the build into exactly the diff --git a/crates/paimon/src/vindex/mod.rs b/crates/paimon/src/vindex/mod.rs index 425a0eec8..f9c6b3355 100644 --- a/crates/paimon/src/vindex/mod.rs +++ b/crates/paimon/src/vindex/mod.rs @@ -524,7 +524,6 @@ mod tests { assert!(!is_vindex_index_type("ivf-hnsw-sq")); assert!(!is_vindex_index_type("")); assert!(!is_vindex_index_type("btree")); - assert!(!is_vindex_index_type("lumina")); assert!(!is_vindex_index_type("IVF-FLAT")); } diff --git a/crates/paimon/src/vindex/pkvector/ann.rs b/crates/paimon/src/vindex/pkvector/ann.rs index 39de1a03a..793906e9b 100644 --- a/crates/paimon/src/vindex/pkvector/ann.rs +++ b/crates/paimon/src/vindex/pkvector/ann.rs @@ -45,10 +45,8 @@ use crate::vindex::range_reader::VindexFileReader; /// /// The most live row ids one ANN segment's mask may hold. /// -/// Mirrors Java `LuminaVectorGlobalIndexReader.toScopedIds`, which refuses an -/// include-set above `Integer.MAX_VALUE` before allocating the dense array the -/// backend is handed. The number is Paimon's own for this exact quantity at this -/// exact seam, not one picked here. +/// Caps mask expansion at `Integer.MAX_VALUE` live row ids, preserving the +/// existing allocation bound before any ranges are inserted. /// /// The limit is on what goes INTO the mask, charged before each insertion, so /// nothing is ever allocated for a span that could not be handed over anyway. That @@ -436,9 +434,9 @@ pub(crate) enum AnnSegmentSource { pub(crate) type SegmentLoader = Box BoxFuture<'static, crate::Result> + Send + Sync>; -/// Production loader for either buffered Lumina data or a range-backed vindex -/// source. It runs before the dedicated-executor score and must not acquire a -/// second search permit; the bucket leaf already holds one across both phases. +/// Production loader for a range-backed vindex source. It runs before scoring +/// on the dedicated executor and must not acquire a second search permit; the +/// bucket leaf already holds one across both phases. pub(crate) type SourceSegmentLoader = Box< dyn Fn(&BucketAnnSegment) -> BoxFuture<'static, crate::Result> + Send + Sync, >; @@ -546,10 +544,7 @@ impl PkVectorAnnSearcher for VindexAnnSearcher { let source_files = segment.source_meta.source_files(); // The live-row mask depends only on the segment's sources, the active set, // the deletion vectors, and the selections — none of which vary by query — - // so it is BUILT once, then cloned into each query's search. Handing every - // query one `Arc` instead would save those clones, but it also moves a - // filtered search from Lumina's per-query scalar calls onto its native batch - // call; worth doing, not here. + // so it is built once, then cloned into each query's search. let live = build_live_row_ids( source_files, active_source_files, @@ -1181,9 +1176,8 @@ mod tests { #[test] fn the_no_prefilter_shape_reaches_the_backend_with_no_filter_at_all() { // The backend-facing half of the no-pre-filter case. `include_row_ids` is - // what Lumina turns into a Vec of every live id and a filtered search; - // for a query that filters nothing it must stay unset, exactly as on the - // manifest route. + // the reader's row-id filter; for a query that filters nothing it must + // stay unset, exactly as on the manifest route. use std::sync::{Arc, Mutex}; let seen: Arc>> = Arc::new(Mutex::new(None)); let scorer_seen = Arc::clone(&seen); @@ -1316,10 +1310,9 @@ mod tests { } #[test] - fn the_live_row_limit_is_javas_dense_filter_limit() { - // Java rejects an include-set above `Integer.MAX_VALUE` before it allocates - // the dense array (`LuminaVectorGlobalIndexReader.toScopedIds`). Exactly that - // many rows is what a backend can still be handed; one more is not. + fn the_live_row_limit_bounds_mask_allocation() { + // Exactly Integer.MAX_VALUE inserted rows fit the allocation bound; + // one more must be rejected before materializing the range. let at_limit = vec![PkVectorSourceFile::new("f0".into(), i32::MAX as i64).unwrap()]; let mut dvs = HashMap::new(); dvs.insert("f0".to_string(), dv(&[0])); diff --git a/crates/paimon/src/vindex/pkvector/metric.rs b/crates/paimon/src/vindex/pkvector/metric.rs index 59aa24c53..7d654e585 100644 --- a/crates/paimon/src/vindex/pkvector/metric.rs +++ b/crates/paimon/src/vindex/pkvector/metric.rs @@ -16,7 +16,6 @@ // under the License. use super::data_invalid; -use crate::lumina::LuminaVectorMetric; use std::cmp::Ordering; /// Order two distances the way Java `Float.compare` does: every NaN sorts after @@ -63,17 +62,6 @@ impl VectorSearchMetric { } } - /// Map a Lumina metric to this enum, symmetric to `from_vindex`. Lets the - /// read path compare the metric a Lumina segment was built with against the - /// configured metric. - pub(crate) fn from_lumina(metric: LuminaVectorMetric) -> Self { - match metric { - LuminaVectorMetric::L2 => Self::L2, - LuminaVectorMetric::Cosine => Self::Cosine, - LuminaVectorMetric::InnerProduct => Self::InnerProduct, - } - } - /// Normalize, validate, and map to the enum. Errors on an unsupported metric. pub(crate) fn parse(metric: &str) -> crate::Result { match normalize_metric(metric).as_str() { @@ -332,21 +320,4 @@ mod tests { VectorSearchMetric::InnerProduct ); } - - #[test] - fn test_from_lumina_maps_every_variant() { - use crate::lumina::LuminaVectorMetric; - assert_eq!( - VectorSearchMetric::from_lumina(LuminaVectorMetric::L2), - VectorSearchMetric::L2 - ); - assert_eq!( - VectorSearchMetric::from_lumina(LuminaVectorMetric::Cosine), - VectorSearchMetric::Cosine - ); - assert_eq!( - VectorSearchMetric::from_lumina(LuminaVectorMetric::InnerProduct), - VectorSearchMetric::InnerProduct - ); - } } diff --git a/crates/paimon/tests/pk_vector_baseline_test.rs b/crates/paimon/tests/pk_vector_baseline_test.rs index c11bb0c7f..9285499ae 100644 --- a/crates/paimon/tests/pk_vector_baseline_test.rs +++ b/crates/paimon/tests/pk_vector_baseline_test.rs @@ -25,11 +25,9 @@ //! (result reader -> Arrow batches, best-first order, `__paimon_search_score`). //! //! Why Rust-built rather than a committed cross-language fixture: the Java -//! primary-key vector ANN segment is an opaque native Lumina format that cannot -//! be reproduced byte-for-byte here, whereas the Rust read path is backed by the -//! vindex (IVF) segment format. This test therefore validates the Rust read path -//! against real vindex IVF segment bytes it produces itself, with no committed -//! binaries and nothing skipped. +//! primary-key vector fixture uses a format that this Rust test does not build. +//! This test validates the Rust read path against real vindex IVF segment bytes +//! it produces itself, with no external binaries or skipped tests. //! //! Two constraints the primary-key read path enforces are satisfied by hand //! (mirroring Java `PrimaryKeyIndexSourcePolicy` and `PrimaryKeyIndexSourceMeta`): diff --git a/docs/src/releases.md b/docs/src/releases.md index 55f8b599f..2742ce8c7 100644 --- a/docs/src/releases.md +++ b/docs/src/releases.md @@ -27,6 +27,8 @@ Apache Paimon Rust follows [Semantic Versioning](https://semver.org/). All relea ### 0.5.0 (In Development) +- Removed Lumina vector index support; use vindex DiskANN via `create_global_index` with `index_type => 'diskann'`. Rebuild existing Lumina indexes in the vindex format. + See [GitHub Issues](https://github.com/apache/paimon-rust/issues) for the current roadmap. ## Past Releases diff --git a/docs/src/sql.md b/docs/src/sql.md index e66918a70..e49c2d225 100644 --- a/docs/src/sql.md +++ b/docs/src/sql.md @@ -1494,9 +1494,8 @@ CALL sys.create_global_index( ); ``` -The vindex `diskann` index is separate from the Lumina index whose -`lumina.index.type` is `diskann`. Use the `diskann.*` options below with -`index_type => 'diskann'`; use `lumina.*` options only with a Lumina index. +For vector search, we recommend vindex DiskANN. Select `index_type => 'diskann'` +and configure it with the `diskann.*` options below. The `options` argument is a comma-separated `key=value` string. User options override table options. Use keys prefixed by the selected index type, or set @@ -1596,7 +1595,7 @@ CALL sys.drop_global_index( ``` `index_type` accepts every type the create procedures build: `btree`, `bitmap`, -`multivalue`, `fm`, `full-text`, `lumina` (or `lumina-vector-ann`), and the vindex types +`multivalue`, `fm`, `full-text`, and the vindex types `ivf-flat`, `ivf-pq`, `ivf-sq`, `ivf-rq`, and `diskann`. It defaults to `btree`, is case-insensitive and surrounding whitespace is ignored. @@ -1623,36 +1622,6 @@ The preview uses the latest snapshot at the time of the call; concurrent changes can affect the number of files matched by a later drop. Reads retain the usual local-cache behavior. Partition filtering is not supported by this procedure. -### create_lumina_index - -Build and commit a Lumina global vector index for a table column: - -```sql -CALL sys.create_lumina_index(table => 'paimon.my_db.my_table', index_column => 'embedding'); -``` - -The optional `index_type` argument selects the Lumina index identifier. It -defaults to `lumina`, is case-insensitive and surrounding whitespace is ignored. -Valid values are `lumina` and the legacy-compatible `lumina-vector-ann`. - -```sql -CALL sys.create_lumina_index( - table => 'paimon.my_db.my_table', - index_column => 'embedding', - index_type => 'lumina' -); -``` - -Optional Lumina builder settings can be supplied as comma-separated `key=value` pairs: - -```sql -CALL sys.create_lumina_index( - table => 'paimon.my_db.my_table', - index_column => 'embedding', - options => 'lumina.index.dimension=128,lumina.encoding.type=pq' -); -``` - ### grant_permission Grant an access to a principal on a REST catalog. Only the REST catalog supports @@ -1905,9 +1874,9 @@ CROSS JOIN LATERAL vector_search( Paimon supports approximate nearest neighbor (ANN) vector search through global vector indexes. DataFusion can search vindex indexes created by -`CALL sys.create_global_index` and Lumina indexes created by -`CALL sys.create_lumina_index`. The `vector_search` table-valued function is -registered as a UDTF on the DataFusion session context. +`CALL sys.create_global_index`. We recommend vindex DiskANN +(`index_type => 'diskann'`). The `vector_search` table-valued function is registered +as a UDTF on the DataFusion session context. ### Registration @@ -2130,22 +2099,6 @@ that dynamic-option path. Rust callers can pass the same unprefixed keys, such as `diskann.l_search`, through `VectorSearchBuilder::with_options` or `BatchVectorSearchBuilder::with_options`. -### Lumina Index Options - -Lumina index behavior is configured via table options prefixed with `lumina.`: - -| Option | Description | -|---|---| -| `lumina.index.dimension` | Vector dimension | -| `lumina.distance.metric` | Distance metric (`inner_product`, `cosine`, `l2`) | -| `lumina.index.type` | Index type (default: `diskann`) | -| `lumina.encoding.type` | Encoding type (default: `pq`) | -| `lumina.search.max-filter-bytes` | Maximum memory used by each active Lumina reader to expand a filtered row-id set (default: `64 MiB`, or 8,388,608 row ids). Queries exceeding it fail with an error; increase it explicitly as a table or vector-search option for unusually large index segments. This Rust-side safety option is not passed to the native Lumina library. | - -### Lumina Environment - -The Lumina native library must be available at runtime. Set the `LUMINA_LIB_PATH` environment variable to the path of the shared library, or place it in the platform default location. - ## Hybrid Search Paimon supports hybrid search by combining multiple search routes and ranking the merged results. The `hybrid_search` table-valued function is registered as a UDTF on the DataFusion session context. @@ -2754,7 +2707,7 @@ Columns: |---|---|---| | `partition` | STRING | Partition spec for the indexed data, formatted as a Java row cast string; `{}` for unpartitioned tables | | `bucket` | INT | Bucket id covered by the index file | -| `index_type` | STRING | Index type, such as `btree`, `bitmap`, `multivalue`, `fm`, `ivf-flat`, `lumina`, or `DELETION_VECTORS` | +| `index_type` | STRING | Index type, such as `btree`, `bitmap`, `multivalue`, `fm`, `ivf-flat`, `diskann`, or `DELETION_VECTORS` | | `file_name` | STRING | Index file name. It resolves to the table `index/` directory, or to the bucket's data-file directory when `index-file-in-data-file-dir` is set; an index file with an external path is read from that path instead | | `file_size` | BIGINT | Index file size in bytes | | `row_count` | BIGINT | Number of rows covered by the index file |