diff --git a/AGENTS.md b/AGENTS.md index a3c14c41..d32a25da 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -16,9 +16,9 @@ Tools that support `@`-imports (Claude Code) auto-include all three files via th `CLAUDE.md` is a symlink to this file — there is exactly one source of truth. Edit `AGENTS.md`. -**Version surveyed:** 0.8.1 +**Version surveyed:** 0.9.0 **Workspace crates:** `omnigraph-compiler`, `omnigraph` (engine), `omnigraph-policy`, `omnigraph-api-types` (shared HTTP wire DTOs), `omnigraph-cluster`, `omnigraph-cli`, `omnigraph-server` -**Storage substrate:** Lance 9.x (columnar, versioned, branchable; 9.0.0-rc.1 git-rev pin until 9.0.0 stable) +**Storage substrate:** Lance 9.0.0 (columnar, versioned, branchable; crates.io release — the 9.x git-rev pin era ended 2026-07-25) **License:** MIT **Toolchain:** Rust stable, edition 2024 diff --git a/Cargo.lock b/Cargo.lock index 5874a26b..6825accc 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2778,8 +2778,9 @@ checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c" [[package]] name = "fsst" -version = "9.0.0-rc.1" -source = "git+https://github.com/lance-format/lance?rev=cec0b7dffe2d85c7e66dbe9d1f3891c297903a1d#cec0b7dffe2d85c7e66dbe9d1f3891c297903a1d" +version = "9.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6f1155128aba964cf6925c22a667ed763b52c8c653693c4b46f8a79d509a8c1" dependencies = [ "arrow-array", "rand 0.9.4", @@ -3795,8 +3796,9 @@ dependencies = [ [[package]] name = "lance" -version = "9.0.0-rc.1" -source = "git+https://github.com/lance-format/lance?rev=cec0b7dffe2d85c7e66dbe9d1f3891c297903a1d#cec0b7dffe2d85c7e66dbe9d1f3891c297903a1d" +version = "9.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23d04bed056e254bc6e31264b031c8492507ca57939586f016924081dcf221a9" dependencies = [ "arc-swap", "arrow", @@ -3869,8 +3871,9 @@ dependencies = [ [[package]] name = "lance-arrow" -version = "9.0.0-rc.1" -source = "git+https://github.com/lance-format/lance?rev=cec0b7dffe2d85c7e66dbe9d1f3891c297903a1d#cec0b7dffe2d85c7e66dbe9d1f3891c297903a1d" +version = "9.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6e7b87c8183988c40a6bd30a6d8ec588b84e53f702b82f19859dc71ba6c02bc" dependencies = [ "arrow-array", "arrow-buffer", @@ -3892,7 +3895,8 @@ dependencies = [ [[package]] name = "lance-arrow-scalar" version = "58.0.0" -source = "git+https://github.com/lance-format/lance?rev=cec0b7dffe2d85c7e66dbe9d1f3891c297903a1d#cec0b7dffe2d85c7e66dbe9d1f3891c297903a1d" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "771f68b04b47f3addf781116f65061808de94b05e1e9411c23c18f32d14ebe79" dependencies = [ "arrow-array", "arrow-buffer", @@ -3906,17 +3910,20 @@ dependencies = [ [[package]] name = "lance-arrow-stats" version = "58.0.0" -source = "git+https://github.com/lance-format/lance?rev=cec0b7dffe2d85c7e66dbe9d1f3891c297903a1d#cec0b7dffe2d85c7e66dbe9d1f3891c297903a1d" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dd47ec33c90bf29f688fd02118e37d3a5ad5c339caa3163f89e417dc0867001f" dependencies = [ "arrow-array", "arrow-schema", + "half", "lance-arrow-scalar", ] [[package]] name = "lance-bitpacking" -version = "9.0.0-rc.1" -source = "git+https://github.com/lance-format/lance?rev=cec0b7dffe2d85c7e66dbe9d1f3891c297903a1d#cec0b7dffe2d85c7e66dbe9d1f3891c297903a1d" +version = "9.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4a4d84a3f36133c70bf89d306d813a29f8eb8555bba2b84995260867cfafdd5e" dependencies = [ "arrayref", "crunchy", @@ -3926,8 +3933,9 @@ dependencies = [ [[package]] name = "lance-core" -version = "9.0.0-rc.1" -source = "git+https://github.com/lance-format/lance?rev=cec0b7dffe2d85c7e66dbe9d1f3891c297903a1d#cec0b7dffe2d85c7e66dbe9d1f3891c297903a1d" +version = "9.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "238c8a58308e7718d6bd96b53494eb7953fa299778bc4911cc571c3576e9446d" dependencies = [ "arrow-array", "arrow-buffer", @@ -3965,8 +3973,9 @@ dependencies = [ [[package]] name = "lance-datafusion" -version = "9.0.0-rc.1" -source = "git+https://github.com/lance-format/lance?rev=cec0b7dffe2d85c7e66dbe9d1f3891c297903a1d#cec0b7dffe2d85c7e66dbe9d1f3891c297903a1d" +version = "9.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "12f0ac4e1cf2f9b1b2fb5ee11bcd04af617bdd6f6cca13726a41c4212220193e" dependencies = [ "arrow", "arrow-array", @@ -3996,8 +4005,9 @@ dependencies = [ [[package]] name = "lance-datagen" -version = "9.0.0-rc.1" -source = "git+https://github.com/lance-format/lance?rev=cec0b7dffe2d85c7e66dbe9d1f3891c297903a1d#cec0b7dffe2d85c7e66dbe9d1f3891c297903a1d" +version = "9.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dd65e7ea88ab28e5d3e91b58a56c02bfd44c47474caae5f8aed1322df1611476" dependencies = [ "arrow", "arrow-array", @@ -4014,8 +4024,9 @@ dependencies = [ [[package]] name = "lance-derive" -version = "9.0.0-rc.1" -source = "git+https://github.com/lance-format/lance?rev=cec0b7dffe2d85c7e66dbe9d1f3891c297903a1d#cec0b7dffe2d85c7e66dbe9d1f3891c297903a1d" +version = "9.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf46656b359786f0b73f13936193583972b7af6e53ccb542da87830be18e94b2" dependencies = [ "proc-macro2", "quote", @@ -4024,8 +4035,9 @@ dependencies = [ [[package]] name = "lance-encoding" -version = "9.0.0-rc.1" -source = "git+https://github.com/lance-format/lance?rev=cec0b7dffe2d85c7e66dbe9d1f3891c297903a1d#cec0b7dffe2d85c7e66dbe9d1f3891c297903a1d" +version = "9.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4725f14fe6cc1b2a5644786b4afa828d0c243064e208fa17378f839243d049c0" dependencies = [ "arrow-arith", "arrow-array", @@ -4060,8 +4072,9 @@ dependencies = [ [[package]] name = "lance-file" -version = "9.0.0-rc.1" -source = "git+https://github.com/lance-format/lance?rev=cec0b7dffe2d85c7e66dbe9d1f3891c297903a1d#cec0b7dffe2d85c7e66dbe9d1f3891c297903a1d" +version = "9.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8c161b00eb5813f98f1d6d52e06f1b712f9eebb0a7f535a8dc1e1122ceca7307" dependencies = [ "arrow-arith", "arrow-array", @@ -4091,8 +4104,9 @@ dependencies = [ [[package]] name = "lance-index" -version = "9.0.0-rc.1" -source = "git+https://github.com/lance-format/lance?rev=cec0b7dffe2d85c7e66dbe9d1f3891c297903a1d#cec0b7dffe2d85c7e66dbe9d1f3891c297903a1d" +version = "9.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf7980b0a6287fd46b9308a31e2cf284065d5693bfb43336297f0400172670ad" dependencies = [ "arc-swap", "arrow", @@ -4157,8 +4171,9 @@ dependencies = [ [[package]] name = "lance-index-core" -version = "9.0.0-rc.1" -source = "git+https://github.com/lance-format/lance?rev=cec0b7dffe2d85c7e66dbe9d1f3891c297903a1d#cec0b7dffe2d85c7e66dbe9d1f3891c297903a1d" +version = "9.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a95f46ac3e4cdd710ca6929226cfb0dd343d5d2370ecb4b40e1c452043f7d27a" dependencies = [ "arrow-array", "arrow-schema", @@ -4180,8 +4195,9 @@ dependencies = [ [[package]] name = "lance-io" -version = "9.0.0-rc.1" -source = "git+https://github.com/lance-format/lance?rev=cec0b7dffe2d85c7e66dbe9d1f3891c297903a1d#cec0b7dffe2d85c7e66dbe9d1f3891c297903a1d" +version = "9.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6deecd351ed6849184cc83000eabb87c8a5bfc519996f92451284498af7b42c" dependencies = [ "arrow", "arrow-arith", @@ -4222,8 +4238,9 @@ dependencies = [ [[package]] name = "lance-linalg" -version = "9.0.0-rc.1" -source = "git+https://github.com/lance-format/lance?rev=cec0b7dffe2d85c7e66dbe9d1f3891c297903a1d#cec0b7dffe2d85c7e66dbe9d1f3891c297903a1d" +version = "9.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e5a9b99bd1f49bc2fe5afb81323141abc506c818f589de95dbab4f6143d9a88" dependencies = [ "arrow-array", "arrow-buffer", @@ -4239,8 +4256,9 @@ dependencies = [ [[package]] name = "lance-namespace" -version = "9.0.0-rc.1" -source = "git+https://github.com/lance-format/lance?rev=cec0b7dffe2d85c7e66dbe9d1f3891c297903a1d#cec0b7dffe2d85c7e66dbe9d1f3891c297903a1d" +version = "9.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8624fd33a894b5f2eb411cb56588d29138137ce100dee8e335843828e249b860" dependencies = [ "arrow", "async-trait", @@ -4252,8 +4270,9 @@ dependencies = [ [[package]] name = "lance-namespace-impls" -version = "9.0.0-rc.1" -source = "git+https://github.com/lance-format/lance?rev=cec0b7dffe2d85c7e66dbe9d1f3891c297903a1d#cec0b7dffe2d85c7e66dbe9d1f3891c297903a1d" +version = "9.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39f0cb9651a65f8eb411d825f34967fff46ba575a76e6ae9becda6ee31c1a974" dependencies = [ "arrow", "arrow-ipc", @@ -4297,8 +4316,9 @@ dependencies = [ [[package]] name = "lance-select" -version = "9.0.0-rc.1" -source = "git+https://github.com/lance-format/lance?rev=cec0b7dffe2d85c7e66dbe9d1f3891c297903a1d#cec0b7dffe2d85c7e66dbe9d1f3891c297903a1d" +version = "9.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb5c718c141ea4f067203b11a54ccf57f8effbf963f345bc811f4837a660ad57" dependencies = [ "arrow-array", "arrow-buffer", @@ -4313,8 +4333,9 @@ dependencies = [ [[package]] name = "lance-table" -version = "9.0.0-rc.1" -source = "git+https://github.com/lance-format/lance?rev=cec0b7dffe2d85c7e66dbe9d1f3891c297903a1d#cec0b7dffe2d85c7e66dbe9d1f3891c297903a1d" +version = "9.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ffc125611b4fe46f99f00b5262e71e17bd947a3bc3b81f3a4a604cc9ca290b3f" dependencies = [ "arrow", "arrow-array", @@ -4351,8 +4372,9 @@ dependencies = [ [[package]] name = "lance-tokenizer" -version = "9.0.0-rc.1" -source = "git+https://github.com/lance-format/lance?rev=cec0b7dffe2d85c7e66dbe9d1f3891c297903a1d#cec0b7dffe2d85c7e66dbe9d1f3891c297903a1d" +version = "9.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73060a844ceda2405759b8f0f16a1c586b5b285dbdf7a3784350d048f991cfd3" dependencies = [ "icu_segmenter", "rust-stemmers", @@ -4988,7 +5010,7 @@ dependencies = [ [[package]] name = "omnigraph-api-types" -version = "0.8.1" +version = "0.9.0" dependencies = [ "omnigraph-compiler", "omnigraph-engine", @@ -4999,7 +5021,7 @@ dependencies = [ [[package]] name = "omnigraph-cli" -version = "0.8.1" +version = "0.9.0" dependencies = [ "assert_cmd", "clap", @@ -5023,7 +5045,7 @@ dependencies = [ [[package]] name = "omnigraph-cluster" -version = "0.8.1" +version = "0.9.0" dependencies = [ "fail", "omnigraph-compiler", @@ -5042,7 +5064,7 @@ dependencies = [ [[package]] name = "omnigraph-compiler" -version = "0.8.1" +version = "0.9.0" dependencies = [ "ahash", "arrow-array", @@ -5062,7 +5084,7 @@ dependencies = [ [[package]] name = "omnigraph-engine" -version = "0.8.1" +version = "0.9.0" dependencies = [ "arc-swap", "arrow-array", @@ -5110,7 +5132,7 @@ dependencies = [ [[package]] name = "omnigraph-policy" -version = "0.8.1" +version = "0.9.0" dependencies = [ "cedar-policy", "clap", @@ -5123,7 +5145,7 @@ dependencies = [ [[package]] name = "omnigraph-server" -version = "0.8.1" +version = "0.9.0" dependencies = [ "arc-swap", "async-trait", diff --git a/Cargo.toml b/Cargo.toml index 99aacc0a..fc8974e4 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -31,21 +31,23 @@ datafusion-common = "54" datafusion-expr = "54" datafusion-functions-aggregate = "54" -# Lance 9.0.0-rc.1 (tag v9.0.0-rc.1 -> rev cec0b7df): the 9.x release -# candidates are git-tags only. rev-pinned for reproducibility. -# Carries lance#7480 (rowid overlap, closes #7444 -> vendor pin retired) and -# lance#7320 (BTREE segment-merge, closes #7230). See docs/dev/lance.md. -lance = { git = "https://github.com/lance-format/lance", rev = "cec0b7dffe2d85c7e66dbe9d1f3891c297903a1d", default-features = false, features = ["aws"] } -lance-core = { git = "https://github.com/lance-format/lance", rev = "cec0b7dffe2d85c7e66dbe9d1f3891c297903a1d" } +# Lance 9.0.0 (crates.io). The 9.x prereleases were git-tags only, so this +# workspace rev-pinned `v9.0.0-rc.1` (cec0b7df) until the stable release +# landed on 2026-07-24; registry versions retire that git pin and restore +# ordinary publishability. Carries lance#7480 (rowid overlap, closes #7444 -> +# vendor pin retired) and lance#7320 (BTREE segment-merge, closes #7230). +# See docs/dev/lance.md. +lance = { version = "9.0.0", default-features = false, features = ["aws"] } +lance-core = "9.0.0" # RowAddrTreeMap moved from lance-core::utils::mask to the new lance-select crate in 9.x. -lance-select = { git = "https://github.com/lance-format/lance", rev = "cec0b7dffe2d85c7e66dbe9d1f3891c297903a1d" } -lance-datafusion = { git = "https://github.com/lance-format/lance", rev = "cec0b7dffe2d85c7e66dbe9d1f3891c297903a1d" } -lance-file = { git = "https://github.com/lance-format/lance", rev = "cec0b7dffe2d85c7e66dbe9d1f3891c297903a1d" } -lance-index = { git = "https://github.com/lance-format/lance", rev = "cec0b7dffe2d85c7e66dbe9d1f3891c297903a1d" } -lance-linalg = { git = "https://github.com/lance-format/lance", rev = "cec0b7dffe2d85c7e66dbe9d1f3891c297903a1d" } -lance-namespace = { git = "https://github.com/lance-format/lance", rev = "cec0b7dffe2d85c7e66dbe9d1f3891c297903a1d" } -lance-namespace-impls = { git = "https://github.com/lance-format/lance", rev = "cec0b7dffe2d85c7e66dbe9d1f3891c297903a1d" } -lance-table = { git = "https://github.com/lance-format/lance", rev = "cec0b7dffe2d85c7e66dbe9d1f3891c297903a1d" } +lance-select = "9.0.0" +lance-datafusion = "9.0.0" +lance-file = "9.0.0" +lance-index = "9.0.0" +lance-linalg = "9.0.0" +lance-namespace = "9.0.0" +lance-namespace-impls = "9.0.0" +lance-table = "9.0.0" ulid = "1" futures = "0.3" diff --git a/crates/omnigraph-api-types/Cargo.toml b/crates/omnigraph-api-types/Cargo.toml index 114c2401..b4bdf0d3 100644 --- a/crates/omnigraph-api-types/Cargo.toml +++ b/crates/omnigraph-api-types/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "omnigraph-api-types" -version = "0.8.1" +version = "0.9.0" edition = "2024" description = "Shared HTTP wire DTOs for Omnigraph — request/response types and engine-result → DTO mappings used by both omnigraph-server and omnigraph-cli (RFC-009). Plain serde/utoipa types; no transport or server internals." license = "MIT" @@ -9,8 +9,8 @@ homepage = "https://github.com/ModernRelay/omnigraph" documentation = "https://docs.rs/omnigraph-api-types" [dependencies] -omnigraph = { package = "omnigraph-engine", path = "../omnigraph", version = "0.8.1" } -omnigraph-compiler = { path = "../omnigraph-compiler", version = "0.8.1" } +omnigraph = { package = "omnigraph-engine", path = "../omnigraph", version = "0.9.0" } +omnigraph-compiler = { path = "../omnigraph-compiler", version = "0.9.0" } serde = { workspace = true } serde_json = { workspace = true } utoipa = { workspace = true } diff --git a/crates/omnigraph-cli/Cargo.toml b/crates/omnigraph-cli/Cargo.toml index f8b7a2ad..a0ab134e 100644 --- a/crates/omnigraph-cli/Cargo.toml +++ b/crates/omnigraph-cli/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "omnigraph-cli" -version = "0.8.1" +version = "0.9.0" edition = "2024" description = "CLI for the Omnigraph graph database." license = "MIT" @@ -13,12 +13,12 @@ name = "omnigraph" path = "src/main.rs" [dependencies] -omnigraph = { package = "omnigraph-engine", path = "../omnigraph", version = "0.8.1" } -omnigraph-compiler = { path = "../omnigraph-compiler", version = "0.8.1" } -omnigraph-api-types = { path = "../omnigraph-api-types", version = "0.8.1" } -omnigraph-cluster = { path = "../omnigraph-cluster", version = "0.8.1" } -omnigraph-policy = { path = "../omnigraph-policy", version = "0.8.1" } -omnigraph-server = { path = "../omnigraph-server", version = "0.8.1" } +omnigraph = { package = "omnigraph-engine", path = "../omnigraph", version = "0.9.0" } +omnigraph-compiler = { path = "../omnigraph-compiler", version = "0.9.0" } +omnigraph-api-types = { path = "../omnigraph-api-types", version = "0.9.0" } +omnigraph-cluster = { path = "../omnigraph-cluster", version = "0.9.0" } +omnigraph-policy = { path = "../omnigraph-policy", version = "0.9.0" } +omnigraph-server = { path = "../omnigraph-server", version = "0.9.0" } clap = { workspace = true } color-eyre = { workspace = true } serde = { workspace = true } diff --git a/crates/omnigraph-cluster/Cargo.toml b/crates/omnigraph-cluster/Cargo.toml index fadac791..1e75df2f 100644 --- a/crates/omnigraph-cluster/Cargo.toml +++ b/crates/omnigraph-cluster/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "omnigraph-cluster" -version = "0.8.1" +version = "0.9.0" edition = "2024" description = "Cluster configuration validation, planning, and config-only apply for Omnigraph." license = "MIT" @@ -14,8 +14,8 @@ documentation = "https://docs.rs/omnigraph-cluster" failpoints = ["dep:fail", "fail/failpoints", "omnigraph/failpoints"] [dependencies] -omnigraph-compiler = { path = "../omnigraph-compiler", version = "0.8.1" } -omnigraph = { package = "omnigraph-engine", path = "../omnigraph", version = "0.8.1" } +omnigraph-compiler = { path = "../omnigraph-compiler", version = "0.9.0" } +omnigraph = { package = "omnigraph-engine", path = "../omnigraph", version = "0.9.0" } fail = { workspace = true, optional = true } serde = { workspace = true } serde_json = { workspace = true } diff --git a/crates/omnigraph-compiler/Cargo.toml b/crates/omnigraph-compiler/Cargo.toml index a8257d9b..4bc06ad0 100644 --- a/crates/omnigraph-compiler/Cargo.toml +++ b/crates/omnigraph-compiler/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "omnigraph-compiler" -version = "0.8.1" +version = "0.9.0" edition = "2024" description = "Schema/query compiler for Omnigraph. Zero Lance dependency." license = "MIT" diff --git a/crates/omnigraph-policy/Cargo.toml b/crates/omnigraph-policy/Cargo.toml index d0e4f111..4fb99e4b 100644 --- a/crates/omnigraph-policy/Cargo.toml +++ b/crates/omnigraph-policy/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "omnigraph-policy" -version = "0.8.1" +version = "0.9.0" edition = "2024" description = "Policy / authorization layer for Omnigraph — Cedar-backed PolicyEngine, PolicyChecker trait, ResourceScope enum." license = "MIT" diff --git a/crates/omnigraph-server/Cargo.toml b/crates/omnigraph-server/Cargo.toml index 5de77321..9ee201fd 100644 --- a/crates/omnigraph-server/Cargo.toml +++ b/crates/omnigraph-server/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "omnigraph-server" -version = "0.8.1" +version = "0.9.0" edition = "2024" description = "HTTP server for the Omnigraph graph database." license = "MIT" @@ -19,11 +19,11 @@ default = [] aws = ["dep:aws-config", "dep:aws-sdk-secretsmanager"] [dependencies] -omnigraph = { package = "omnigraph-engine", path = "../omnigraph", version = "0.8.1" } -omnigraph-compiler = { path = "../omnigraph-compiler", version = "0.8.1" } -omnigraph-policy = { path = "../omnigraph-policy", version = "0.8.1" } -omnigraph-api-types = { path = "../omnigraph-api-types", version = "0.8.1" } -omnigraph-cluster = { path = "../omnigraph-cluster", version = "0.8.1" } +omnigraph = { package = "omnigraph-engine", path = "../omnigraph", version = "0.9.0" } +omnigraph-compiler = { path = "../omnigraph-compiler", version = "0.9.0" } +omnigraph-policy = { path = "../omnigraph-policy", version = "0.9.0" } +omnigraph-api-types = { path = "../omnigraph-api-types", version = "0.9.0" } +omnigraph-cluster = { path = "../omnigraph-cluster", version = "0.9.0" } axum = { workspace = true } clap = { workspace = true } color-eyre = { workspace = true } diff --git a/crates/omnigraph/Cargo.toml b/crates/omnigraph/Cargo.toml index 44490a2d..032947c8 100644 --- a/crates/omnigraph/Cargo.toml +++ b/crates/omnigraph/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "omnigraph-engine" -version = "0.8.1" +version = "0.9.0" edition = "2024" description = "Runtime engine for the Omnigraph graph database." license = "MIT" @@ -16,8 +16,8 @@ default = [] failpoints = ["dep:fail", "fail/failpoints"] [dependencies] -omnigraph-compiler = { path = "../omnigraph-compiler", version = "0.8.1" } -omnigraph-policy = { path = "../omnigraph-policy", version = "0.8.1" } +omnigraph-compiler = { path = "../omnigraph-compiler", version = "0.9.0" } +omnigraph-policy = { path = "../omnigraph-policy", version = "0.9.0" } lance = { workspace = true } lance-core = { workspace = true } lance-select = { workspace = true } @@ -54,12 +54,12 @@ chrono = { workspace = true } arc-swap = { workspace = true } [dev-dependencies] -omnigraph-compiler = { path = "../omnigraph-compiler", version = "0.8.1" } +omnigraph-compiler = { path = "../omnigraph-compiler", version = "0.9.0" } tokio = { workspace = true } lance-namespace-impls = { workspace = true } # test-util gates IoStats.requests + assert_io_eq! (failure diagnostics only); dev-dep, # so production builds (which exclude dev-deps) never see it. -lance-io = { git = "https://github.com/lance-format/lance", rev = "cec0b7dffe2d85c7e66dbe9d1f3891c297903a1d", features = ["test-util"] } +lance-io = { version = "9.0.0", features = ["test-util"] } serial_test = "3" proptest = "1" syn = { version = "2", features = ["full", "visit"] } diff --git a/crates/omnigraph/src/db/manifest/migrations.rs b/crates/omnigraph/src/db/manifest/migrations.rs index 8522e3b7..146eb55c 100644 --- a/crates/omnigraph/src/db/manifest/migrations.rs +++ b/crates/omnigraph/src/db/manifest/migrations.rs @@ -92,19 +92,28 @@ pub(crate) const MIN_SUPPORTED_INTERNAL_SCHEMA_VERSION: u32 = INTERNAL_MANIFEST_ /// see `docs/user/operations/upgrade.md`). Ranges are the release tags that /// stamped each version (verify with /// `git show vX.Y.Z:crates/omnigraph/src/db/manifest/migrations.rs`): -/// v1 ≤ 0.3.1, v2 0.4.1–0.6.1, v3 0.6.2–0.7.2, v4 0.8.x, v5 0.9.x, -/// v6 0.10.x, v7 0.11.x, v8 0.12.x, v9 0.13.x. +/// v1 ≤ 0.3.1, v2 0.4.1–0.6.1, v3 0.6.2–0.7.2, v4 0.8.x, v5–v8 unreleased, +/// v9 0.9.x. +/// +/// v5 through v8 never reached a published release: the format advanced five +/// times (RFC-028 identity, RFC-023 key fencing, and the three RFC-026 stream +/// slices) inside the single 0.8.1 → 0.9.0 development window, so the only +/// graphs carrying those stamps came from source builds off `main`. An earlier +/// revision of this table optimistically assigned each of them its own release +/// line (0.9.x–0.12.x); those releases do not exist and naming them here would +/// send an operator hunting for a binary that was never published. pub(crate) fn release_for_internal_schema_version(stamp: u32) -> &'static str { match stamp { 1 => "0.3.1 or earlier", 2 => "0.4.1 to 0.6.1", 3 => "0.6.2 to 0.7.2", 4 => "0.8.x", - 5 => "0.9.x", - 6 => "0.10.x", - 7 => "0.11.x", - 8 => "0.12.x", - 9 => "0.13.x", + // Reads in both message slots ("created by omnigraph X" and "with an + // omnigraph X binary"). No such binary was ever published, so + // upgrade.md explains that these graphs must be exported with the + // source commit that stamped them. + 5..=8 => "0.9.0-dev", + 9 => "0.9.x", // Unreachable today (1–9 are mapped; > CURRENT is caught by the ceiling // guard before this is consulted). Worded to read naturally after // "created by omnigraph " if a future bump ever leaves a gap. @@ -216,11 +225,12 @@ mod tests { fn release_names_the_writing_line_for_each_stamp() { assert_eq!(release_for_internal_schema_version(3), "0.6.2 to 0.7.2"); assert_eq!(release_for_internal_schema_version(4), "0.8.x"); - assert_eq!(release_for_internal_schema_version(5), "0.9.x"); - assert_eq!(release_for_internal_schema_version(6), "0.10.x"); - assert_eq!(release_for_internal_schema_version(7), "0.11.x"); - assert_eq!(release_for_internal_schema_version(8), "0.12.x"); - assert_eq!(release_for_internal_schema_version(9), "0.13.x"); + // v5-v8 advanced and were superseded entirely within the 0.8.1 -> 0.9.0 + // development window, so no published release stamped them. + for unreleased in 5..=8 { + assert_eq!(release_for_internal_schema_version(unreleased), "0.9.0-dev"); + } + assert_eq!(release_for_internal_schema_version(9), "0.9.x"); assert_eq!( release_for_internal_schema_version(99), "an unrecognized older release" @@ -231,7 +241,13 @@ mod tests { assert!(err.contains("omnigraph export"), "got: {err}"); let v6_err = refuse_if_stamp_unsupported(6).unwrap_err().to_string(); - assert!(v6_err.contains("0.10.x"), "got: {v6_err}"); + assert!(v6_err.contains("0.9.0-dev"), "got: {v6_err}"); assert!(v6_err.contains("omnigraph export"), "got: {v6_err}"); + // The embedded release must read naturally in both slots of the + // rebuild instruction, not just the "created by" clause. + assert!( + v6_err.contains("with an omnigraph 0.9.0-dev binary"), + "got: {v6_err}" + ); } } diff --git a/crates/omnigraph/src/db/manifest/tests.rs b/crates/omnigraph/src/db/manifest/tests.rs index 6039ab6a..a1d0cfef 100644 --- a/crates/omnigraph/src/db/manifest/tests.rs +++ b/crates/omnigraph/src/db/manifest/tests.rs @@ -2398,9 +2398,11 @@ async fn sub_current_graph_is_refused_then_rebuilt_via_export_import() { "the refusal must nudge the operator to `omnigraph export`, got: {err}", ); assert!( - msg.contains("0.10.x"), - "the refusal must name the release that wrote this stamp (v6 → 0.10.x) so the \ - operator knows which binary to use, got: {err}", + msg.contains("0.9.0-dev"), + "the refusal must name the build that wrote this stamp so the operator knows \ + which binary to use. v6 never shipped in a release — it existed only in \ + source builds during 0.9.0 development — so the refusal names `0.9.0-dev`, \ + got: {err}", ); // Rebuild with this binary: fresh init + load the export. diff --git a/crates/omnigraph/src/table_store.rs b/crates/omnigraph/src/table_store.rs index d8a4d184..00162192 100644 --- a/crates/omnigraph/src/table_store.rs +++ b/crates/omnigraph/src/table_store.rs @@ -2726,22 +2726,21 @@ impl TableStore { /// surface twice — once via the original committed fragment, once via /// the rewrite in `new_fragments`. /// - /// **Filter contract is incomplete on staged fragments.** When `filter` - /// is `Some(...)`, Lance pushes the predicate to per-fragment scans - /// with stats-based pruning. Uncommitted fragments produced by - /// `write_fragments_internal` lack the per-column statistics that - /// committed fragments carry; Lance's optimizer drops them from the - /// filtered scan even when their data would match. Staged-fragment - /// rows are silently absent from the result. `scanner.use_stats(false)` - /// does not fix this in lance 6.0.1. Callers needing correct filtered - /// reads against staged data should use a different strategy — the - /// engine's `MutationStaging` accumulator unions in-memory pending - /// batches with the committed scan via DataFusion `MemTable` (see - /// `scan_with_pending`). + /// **Filtered staged reads were incomplete before Lance 9.0.0.** Through + /// 9.0.0-rc.1, a `Some(filter)` scan pushed the predicate to per-fragment + /// scans with stats-based pruning, and uncommitted fragments produced by + /// `write_fragments_internal` lack the per-column statistics committed + /// fragments carry — so Lance's optimizer dropped them from the filtered + /// scan even when their data matched, and `scanner.use_stats(false)` did + /// not bypass it. Lance 9.0.0 closed that gap: matching staged rows are + /// now returned. `staged_tests::scan_with_staged_with_filter_returns_ + /// matching_staged_rows` pins the current behavior. /// - /// This method remains on the surface for primitive-level testing - /// (basic stage + scan correctness without filters works) and for - /// callers that don't need filter pushdown. + /// Production never depended on either side of this. The engine's + /// `MutationStaging` accumulator unions in-memory pending batches with + /// the committed scan via DataFusion `MemTable` for read-your-writes + /// (see `scan_with_pending`), and no production caller passes a filter + /// here. This method remains on the surface for primitive-level testing. pub async fn scan_with_staged( &self, ds: &Dataset, diff --git a/crates/omnigraph/src/table_store/staged_tests.rs b/crates/omnigraph/src/table_store/staged_tests.rs index 6f713300..52fe741f 100644 --- a/crates/omnigraph/src/table_store/staged_tests.rs +++ b/crates/omnigraph/src/table_store/staged_tests.rs @@ -1948,21 +1948,23 @@ async fn exact_overwrite_rejects_foreign_append_without_altering_the_winner() { ); } -/// **Documented limitation** (see `scan_with_staged` doc): when a filter -/// is supplied, Lance's stats-based pruning drops the staged fragment from -/// the filtered scan because uncommitted fragments produced by -/// `write_fragments_internal` lack per-column statistics. The result -/// contains only matching committed rows; matching staged rows are -/// silently absent. `scanner.use_stats(false)` does not bypass this in -/// lance 6.0.1. +/// **Limitation closed at the Lance 9.0.0 bump.** Through 9.0.0-rc.1, a +/// filtered `scan_with_staged` silently dropped matching staged rows: +/// Lance's stats-based pruning discarded the staged fragment because +/// uncommitted fragments produced by `write_fragments_internal` lack +/// per-column statistics, and `scanner.use_stats(false)` did not bypass +/// it. This test pinned that behavior with an explicit instruction to +/// rewrite it if Lance ever scanned uncommitted fragments without +/// stats-based pruning. /// -/// This test pins the actual behavior so a future change either -/// preserves it (and updates the doc) or fixes it (and rewrites this -/// test). The engine's `MutationStaging` accumulator unions in-memory -/// pending batches with the committed scan via DataFusion `MemTable` -/// for read-your-writes instead, so production code is unaffected. +/// Lance 9.0.0 does exactly that, so the assertion is now the correct +/// semantics: a filtered staged scan returns matching committed *and* +/// staged rows. Production was never affected either way — the engine's +/// `MutationStaging` accumulator unions in-memory pending batches with +/// the committed scan via DataFusion `MemTable` for read-your-writes, and +/// no production caller uses `scan_with_staged` with a filter. #[tokio::test] -async fn scan_with_staged_with_filter_silently_drops_staged_rows() { +async fn scan_with_staged_with_filter_returns_matching_staged_rows() { let dir = tempfile::tempdir().unwrap(); let uri = format!("{}/people.lance", dir.path().to_str().unwrap()); let store = TableStore::new(dir.path().to_str().unwrap(), test_session()); @@ -1985,21 +1987,20 @@ async fn scan_with_staged_with_filter_silently_drops_staged_rows() { .await .unwrap(); - // Filter: age >= 30. Correct semantics would return alice, carol, dave. - // Actual: dave (staged, age=35) is dropped — only the committed matches - // come back. + // Filter: age >= 30 must return every match, committed or staged — + // alice and carol from the committed image plus the staged dave. let batches = store .scan_with_staged(&ds, std::slice::from_ref(&staged), None, Some("age >= 30")) .await .unwrap(); assert_eq!( collect_ids(&batches), - vec!["alice", "carol"], - "documented limitation: filter pushdown drops staged fragments. \ - If you're here because this assertion failed: either (a) Lance \ - exposed a way to scan uncommitted fragments without stats-based \ - pruning (good — update to assert == [alice, carol, dave]), or \ - (b) something changed in our scan_with_staged path." + vec!["alice", "carol", "dave"], + "a filtered staged scan must return matching staged rows. Through \ + 9.0.0-rc.1 stats-based pruning dropped the staged fragment and this \ + asserted [alice, carol]; Lance 9.0.0 closed that gap. A regression \ + here means either Lance reintroduced the pruning or our \ + scan_with_staged path changed." ); // Without filter, staged data IS visible — confirms the issue is diff --git a/crates/omnigraph/tests/memwal_stream_cost.rs b/crates/omnigraph/tests/memwal_stream_cost.rs index 5997ed53..d0615dd4 100644 --- a/crates/omnigraph/tests/memwal_stream_cost.rs +++ b/crates/omnigraph/tests/memwal_stream_cost.rs @@ -60,7 +60,16 @@ const NO_AUTO_ROLL_ROWS: usize = 8_193; const NO_AUTO_ROLL_BATCHES: usize = 8_193; const RSS_CHILD_ENV: &str = "OMNIGRAPH_MEMWAL_COST_CHILD"; -const SURVEYED_LANCE_REV: &str = "cec0b7dffe2d85c7e66dbe9d1f3891c297903a1d"; +/// The exact Lance release Gate R0's source audit was performed against. +/// Through the 9.0.0-rc.1 era this was a git rev (`cec0b7df`); the 9.0.0 +/// stable bump moved every Lance crate to the crates.io registry, so the +/// tripwire now pins the released version instead. Its purpose is unchanged: +/// the audit must not silently outlive the source it surveyed. +const SURVEYED_LANCE_VERSION: &str = "9.0.0"; +/// Two members of the Lance repository family are versioned against Arrow +/// rather than Lance, so they carry their own surveyed version. +const SURVEYED_LANCE_ARROW_VERSION: &str = "58.0.0"; +const SURVEYED_LANCE_ARROW_PACKAGES: [&str; 2] = ["lance-arrow-scalar", "lance-arrow-stats"]; fn classify_mem_wal_object( components: &[&str], @@ -1729,9 +1738,7 @@ async fn gate_r0_current_inventory_refuses_listed_shard_without_manifest() { #[test] fn gate_r0_source_audit_revision_tripwire_matches_surveyed_lance() { - let expected_source = format!( - "git+https://github.com/lance-format/lance?rev={SURVEYED_LANCE_REV}#{SURVEYED_LANCE_REV}" - ); + let expected_source = "registry+https://github.com/rust-lang/crates.io-index"; let expected_lance_repository_packages = BTreeSet::from([ "fsst", "lance", @@ -1772,13 +1779,28 @@ fn gate_r0_source_audit_revision_tripwire_matches_surveyed_lance() { let Some(source) = value("source") else { continue; }; - if source.starts_with("git+https://github.com/lance-format/lance?") { - assert_eq!( - source, expected_source, - "Lance-repository package {name} moved or became mixed; Gate R0's source audit must be rerun" - ); - observed.entry(name).or_default().push(source); + if !expected_lance_repository_packages.contains(name) { + continue; } + assert!( + !source.starts_with("git+"), + "Lance-repository package {name} is pinned to a git source again; Gate R0's source audit must be rerun against it" + ); + assert_eq!( + source, expected_source, + "Lance-repository package {name} left the crates.io registry; Gate R0's source audit must be rerun" + ); + let version = value("version").expect("every locked package declares a version"); + let expected_version = if SURVEYED_LANCE_ARROW_PACKAGES.contains(&name) { + SURVEYED_LANCE_ARROW_VERSION + } else { + SURVEYED_LANCE_VERSION + }; + assert_eq!( + version, expected_version, + "audited Lance-repository package {name} moved off the surveyed version; Gate R0's source audit must be rerun" + ); + observed.entry(name).or_default().push(version); } assert_eq!( observed.keys().copied().collect::>(), @@ -1786,9 +1808,14 @@ fn gate_r0_source_audit_revision_tripwire_matches_surveyed_lance() { "the exact audited Lance-repository package family changed; Gate R0's source audit must be rerun" ); for package in &expected_lance_repository_packages { + let expected_version = if SURVEYED_LANCE_ARROW_PACKAGES.contains(package) { + SURVEYED_LANCE_ARROW_VERSION + } else { + SURVEYED_LANCE_VERSION + }; assert_eq!( observed.get(package).map(Vec::as_slice), - Some([expected_source.as_str()].as_slice()), + Some([expected_version].as_slice()), "audited Lance-repository package {package} moved, disappeared, or became mixed; Gate R0's source audit must be rerun" ); } @@ -1804,9 +1831,13 @@ fn gate_r0_source_audit_revision_tripwire_matches_surveyed_lance() { "Gate R0 must find exactly one direct source declaration for {package}" ); assert!( - lines[0].contains("git = \"https://github.com/lance-format/lance\"") - && lines[0].contains(&format!("rev = \"{SURVEYED_LANCE_REV}\"")), - "direct Lance manifest pin moved without rerunning Gate R0: {}", + !lines[0].contains("git = "), + "direct Lance manifest pin returned to a git source without rerunning Gate R0: {}", + lines[0] + ); + assert!( + lines[0].contains(&format!("\"{SURVEYED_LANCE_VERSION}\"")), + "direct Lance manifest pin moved off the surveyed version without rerunning Gate R0: {}", lines[0] ); }; diff --git a/docs/dev/lance.md b/docs/dev/lance.md index 55b438e0..ade8015b 100644 --- a/docs/dev/lance.md +++ b/docs/dev/lance.md @@ -156,7 +156,89 @@ If a future need pulls one of these into scope, add a row to the matching domain When Lance ships a major release that changes any of the above (file format bump, new index type, transaction semantics change, new branching primitive), refresh this index in the same change as the omnigraph upgrade. Stale Lance pointers are worse than no pointers. -### Last alignment audit: 2026-07-17 (Lance 9.0.0-rc.1; git rev `cec0b7df`) +### Last alignment audit: 2026-07-25 (Lance 9.0.0 stable; crates.io) + +**The git-pin era is over.** Lance 9.0.0 was released on 2026-07-24 and every +crate OmniGraph depends on is published to crates.io at that version, so the +workspace moved from the `v9.0.0-rc.1` git rev (`cec0b7df`) to ordinary +registry version pins. `Cargo.lock` now contains **zero** git sources. This +restores ordinary publishability: the release gate recorded in +[versioning.md](versioning.md) — "registry publication resumes when Lance 9.0.0 +ships stable" — is satisfied. + +The delta is 35 commits and unusually cheap. It is **not** a storage-format +event: there is no file-format or minimum-reader-version movement, OmniGraph +still writes explicit stable V2_2, and every MemWAL format-bearing file +(`wal.rs`, `batch_store.rs`, `system_index/mem_wal.rs`, `protos/table.proto`, +`mem_wal/util.rs`) is byte-identical to the rc.1 rev. Dependency floors are +unchanged (Arrow 58, DataFusion 54, `object_store` 0.13.2, roaring 0.11.4, +Rust 1.91+); this audit ran on Rust 1.95. + +Behavior-affecting findings: + +- **`ShardWriter::close` now propagates final flush failures** (#7769). This + was the pre-registered re-audit item parked at the rc.1 stanza below; the + contract sentence there is now rewritten. **No code change was required** — + OmniGraph never treats `close` as durability evidence, and the enrollment + adapter already maps its empty-shard close error. The matching lines in + [testing.md](testing.md) and RFC-026 were reworded in the same change. +- **Filtered scans over staged fragments are fixed.** Through rc.1, a + `Some(filter)` `scan_with_staged` silently dropped matching *staged* rows + because stats-based pruning discarded uncommitted fragments that lack + per-column statistics. 9.0.0 returns them. This turned the + documented-limitation pin + `staged_tests::scan_with_staged_with_filter_silently_drops_staged_rows` red — + by design; that test carried an explicit instruction to rewrite it on this + exact event. It is now + `scan_with_staged_with_filter_returns_matching_staged_rows`, asserting the + correct semantics. Production was never affected: no production caller passes + a filter, and read-your-writes goes through `MutationStaging`'s in-memory + union instead. +- **`read_transaction_by_version` no longer populates session caches** + (#7817), adding one ranged read per inline transaction on the RFC-023 + certificate-chain path. The `write_cost` / `warm_read_cost` / `merge_cost` + budgets were re-run and all pass unchanged. +- **BM25 corpus statistics changed** (#7699 drops zero-token documents), which + can legitimately move score ties. The pinned fused ordering in `search.rs` + still holds on this release. +- **Neither RFC-026 upstream ask landed.** `InitializeMemWalBuilder::execute` + is still `-> Result<()>` with an internal commit and no enrollment receipt or + reversible shard-admission seal, and `WalAppender::append` still performs no + post-success writer epoch recheck. Both negative guards + (`mem_wal_deleted_fence_slot_allows_stale_writer_success_on_pinned_lance`, + `cleanup_old_versions_does_not_reclaim_mem_wal_objects`) therefore stay green, + and OmniGraph's adapter-side containment remains load-bearing. +- **Gate R0's revision tripwire was rewritten, not retired.** It pinned the + exact git rev in `Cargo.lock`; it now pins the released version across the + whole Lance package family (with the two Arrow-versioned members carrying + their own surveyed version) and additionally refuses any return to a git + source. Its purpose is unchanged — the source audit must not silently outlive + the source it surveyed — and its underlying RC.1 no-go facts survive because + the MemWAL flush ordering and system-index files did not change. + +Evidence re-run for this bump: 26 Lance surface guards, `cargo test --workspace +--locked` (75 suites), 141 failpoints, 35 `memwal_stream` cells, the +`omnigraph-server --features aws` suite, and the local cost gates. The +bucket-gated RustFS/S3 cells were not run locally and remain CI's +responsibility. + +**Known gaps carried by choosing 9.0.0 over the v10 beta line** (both fixes +exist only on v10, which is 96 commits ahead *and* 36 behind 9.0.0 after a +9.1→10.0 renumber, ships two breaking changes, and renames the MemWAL +vocabulary OmniGraph's recovery sidecars bind to): + +- **#7704** — stable-row-id `filter_deleted_ids` alignment. OmniGraph sets + `enable_stable_row_ids: true` everywhere, stages deletes, and calls + `optimize_indices`, so this is a live exposure. +- **#7868** — flat-KNN ordering, which upstream itself labels Critical. +- **#7965** — blob compaction misclassifying a valid empty blob as NULL, plus + a blob-v1 path where one empty blob can zero out neighbouring payloads. + `omnigraph optimize` compacts blob-bearing tables, so this is reachable. + Note OmniGraph's **own** blob descriptor decoder replicates the same + empty-vs-null misclassification independently of Lance; that is tracked as a + separate defect to fix on its own merits. + +### Prior alignment audit: 2026-07-17 (Lance 9.0.0-rc.1; git rev `cec0b7df`) The pin advanced from `v9.0.0-beta.21` (`1aec1465`) to `v9.0.0-rc.1` (`cec0b7dffe2d85c7e66dbe9d1f3891c297903a1d`) after reviewing @@ -415,8 +497,15 @@ stored post-tombstone Arrow batch memory, including replayed duplicate batches. An empty valid reopen may admit; a non-empty valid replay is routed fold-only. `MemTableStats::estimated_size` has different accounting and does not replace that contract. Retirement first stops the serialized worker and then uses -public `ShardWriter::abort`; `close` ignores final completion errors and is not -durability evidence. +public `ShardWriter::abort`. **Since 9.0.0, `close` propagates final flush, +freeze, and task-shutdown failures** (upstream #7769, merged after the rc.1 pin +and consumed by the 9.0.0 bump); through 9.0.0-rc.1 it discarded those results +and returned a false `Ok(())`. OmniGraph's posture is unchanged under either +contract — **close is never durability evidence**: the B1 worker retires via +`abort`, and the enrollment adapter's two `close` calls are an error-path +best-effort (result deliberately discarded) and a provably-empty-shard close +after `check_fenced` whose error was already mapped, so the stricter contract +can only surface more failures where OmniGraph already fails closed. Behavior-affecting findings in this audit: diff --git a/docs/dev/testing.md b/docs/dev/testing.md index d806ca5e..dd068cfb 100644 --- a/docs/dev/testing.md +++ b/docs/dev/testing.md @@ -117,7 +117,9 @@ The implemented coverage keeps the boundaries split as follows: Two-handle claim/eviction races prove one owner and no eviction past an in-flight waiter. Retirement closes the worker to puts before public `ShardWriter::abort`, and no test treats `ShardWriter::close` as durability - evidence. One background-owned abort completion is retained in the retired + evidence — a posture that holds regardless of upstream's contract, which + tightened in Lance 9.0.0 (#7769) to propagate final flush failures instead of + returning a false `Ok(())`. One background-owned abort completion is retained in the retired entry; a caller deadline never cancels that future, retries abort, or permits reopen. A stalled-handler/deadline/second-retirement test pins that exact RC.1 `shutdown_all` hazard. A claim-vs-drain diff --git a/docs/releases/v0.9.0.md b/docs/releases/v0.9.0.md new file mode 100644 index 00000000..5782f5f3 --- /dev/null +++ b/docs/releases/v0.9.0.md @@ -0,0 +1,135 @@ +# Omnigraph v0.9.0 + +A substrate and durability release. The storage substrate reaches **Lance +9.0.0 stable**, which restores crates.io publication; the write path gains +stable schema identity across renames, substrate-native key-conflict fencing, +and crash recovery for every writer. + +**This release changes the on-disk format.** Internal schema advances from v4 +(0.8.x) to v9, so **v0.8.x graphs must be rebuilt via export/import** — see +[Upgrade notes](#upgrade-notes) below and +[docs/user/operations/upgrade.md](../user/operations/upgrade.md). + +**Channel note:** registry publication resumes, as promised in the v0.8.1 +notes. `Cargo.lock` now carries zero git sources, so `cargo install +omnigraph-cli` works again alongside the binaries, Homebrew, and installer +channels. + +## Highlights + +- **Storage substrate: Lance 9.0.0 stable.** The 9.x prerelease git-rev pin is + retired for ordinary registry versions. This is not a storage-format event on + the substrate side — Omnigraph still writes explicit stable V2_2 files, and + dependency floors are unchanged (Arrow 58, DataFusion 54, `object_store` + 0.13.2). The bump was validated against a full review of the 35 upstream + commits since the pinned release candidate plus the complete local and CI + test matrix. + +- **Stable schema identity across renames.** Type and property identity now + survives a supported rename: renaming a type is a metadata-only migration + that keeps the same dataset, path, Lance version history, and built indexes. + Drop-and-re-add remains a new logical lifetime with a fresh identity and an + independent version sequence. Identity is never inferred from a name, path, + or Lance version. + +- **Key-conflict fencing on keyed writes.** Every node and edge table declares + its exact non-null `id` as the substrate's primary key, and every production + insert and upsert is fenced on that field. Strict inserts that collide with + an existing key now fail with a typed `KeyConflict` instead of depending on + incidental behavior. Merging a branch whose history is provably + insert-only also takes a faster path that stages immutable fragments directly + rather than joining against the merge target. + +- **Crash recovery for every writer.** Mutations, loads, schema applies, branch + merges, index materialization, and `optimize` each record a durable recovery + intent before their first durable effect. An interrupted write is rolled + forward or compensated on the next read-write open, all-or-nothing, and + ambiguous state fails closed rather than being silently adopted. Interrupted + operations surface as a typed `RecoveryRequired` outcome. + +- **`branch merge --delete-branch`.** Deletes the source branch as part of a + successful merge, so the common review-branch workflow no longer needs a + second command. Also available as a field on the branch-merge endpoint. + +- **Container images on release.** `omnigraph-server` images are published to + both GitHub Container Registry and Docker Hub as part of the release + workflow. + +- **Faster uniqueness checking on bulk writes.** Committed `@unique` constraint + probes are batched per constraint group — one filtered scan per chunk of up + to 8,192 keys instead of one scan per row. Write-path I/O for uniqueness + checks is now flat in the number of rows being written. + +- **Streaming-ingest foundation.** The format work behind + [RFC-026](../rfcs/0026-memwal-streaming-ingest.md) — durable write-ahead + generations, lifecycle authority, and atomic fold publication — accounts for + most of the internal-schema movement in this release. It is a physical + correctness foundation only: there is no schema, CLI, HTTP, or SDK surface + for streaming ingest in v0.9.0, and none is enabled by upgrading. + +## Behavior changes + +- **Commit listings are newest-first.** `omnigraph commit list`, `GET + /commits`, and the SDK now return a branch's reachable history most recent + first, matching the documented contract (the implementation had shipped + ascending). Omitting `branch` lists main's history — the description + previously claimed "across all branches", which has been stale since graph + lineage moved into the manifest. + +- **`load --mode append` is strict on existing ids.** An append whose batch + contains an id already present in the table is rejected with a key conflict + rather than absorbed. Use `--mode merge` for upsert semantics. + +- **CLI scope flags are validated per verb.** Flags that a command never reads + are now rejected loudly instead of silently ignored: `--profile` on verbs + that resolve no scope (`init`, the `cluster` family, local-only verbs), + `--store` on `init` (which takes a positional target), and `--as` on + read-only control verbs (only `cluster apply` and `cluster approve` attribute + an actor). Scripts passing a flag that was previously discarded will now get + an error. The ambient `$OMNIGRAPH_PROFILE` default is still ignored by those + verbs rather than rejected. + +- **`graphs list` resolves against the server registry.** It no longer routes + through graph-addressed resolution, which required `--graph` before the + enumeration could run and corrupted the request path when one was supplied. + A scope's `default_graph` is deliberately ignored on this path. + +- **`GET /commits` with no `branch` authorizes as `main`.** The handler already + executed the omitted form as main's history but authorized it with no branch + in scope, so under default-deny a branch-scoped read grant could permit + `?branch=main` while denying the equivalent plain request. The omitted form + now behaves exactly like the explicit `?branch=main` form. + +## Fixes + +- A drain request naming a shard that belongs to another table is refused with + a typed error instead of panicking. + +## Upgrade notes + +- **Storage-format change — rebuild required.** Opening a v0.8.x graph is + refused with a message naming the release line that wrote it and the exact + commands. The recipe: export with a 0.8.x binary, `init` a **different** root + with 0.9.x, then `load --mode overwrite`. Data, vectors, and blobs are + preserved; commit history and branches are not. Keep the old root unchanged + through the rollback window, and never force-init the old root in place. +- **Server deployments:** take the graph out of the serving set, rebuild it + offline with the CLI, then repoint the cluster with `cluster apply`. +- **Internal schema stamps v5 through v8 were never published.** The format + advanced five times inside the single 0.8.1 → 0.9.0 development window, so + only source builds off `main` can carry them; the refusal message names them + `0.9.0-dev`. A published binary only ever wrote v4 (0.8.x) or v9 (0.9.x). +- Downgrade to 0.8.x is not possible for a v9 graph — a 0.8.x binary refuses it, + by design. + +## Developer-facing + +- The workspace builds entirely from registry dependencies again; a Lance + source guard fails if a git source returns. +- The write path is one protocol: every graph-content, schema, and maintenance + transition becomes authoritative at a single manifest publish, and writer + adapters supply exact pre-minted transaction identities rather than inferring + ownership from version movement. See + [docs/dev/writes.md](../dev/writes.md). +- Cross-version rebuild tests run against genuine older binaries rather than + simulated stamps, including the immediately preceding format. diff --git a/docs/rfcs/0026-memwal-streaming-ingest.md b/docs/rfcs/0026-memwal-streaming-ingest.md index c930979f..c61e77e1 100644 --- a/docs/rfcs/0026-memwal-streaming-ingest.md +++ b/docs/rfcs/0026-memwal-streaming-ingest.md @@ -2099,9 +2099,12 @@ keeps it retired and admission closed and returns typed `RecoveryRequired`; B1 never issues a second abort or reopens beside a possibly active handler. A process restart reclassifies the durable state before admission. The caller-quiesce precondition is -structurally guarded. Lance `ShardWriter::close` intentionally ignores final -WAL/frozen-flush completion errors, so `close() == Ok(())` is not durability -evidence and B1 never uses it for retirement. The private implementation has +structurally guarded. B1 never uses `ShardWriter::close` for retirement and +never treats `close() == Ok(())` as durability evidence. That posture is +independent of the upstream contract, which tightened at the Lance 9.0.0 bump: +through 9.0.0-rc.1 `close` discarded final WAL/frozen-flush completion errors +and returned a false `Ok(())`, while 9.0.0 (upstream #7769) propagates them. +The stricter contract can only surface failures B1 already fails closed on. The private implementation has explicit resident-writer, reservation, eviction, and durability-deadline values, but their promotion as product defaults remains gated on the checked-in RSS/latency/backpressure evidence in §12.3. Lance's soft diff --git a/docs/user/operations/upgrade.md b/docs/user/operations/upgrade.md index a4020856..ef154776 100644 --- a/docs/user/operations/upgrade.md +++ b/docs/user/operations/upgrade.md @@ -17,9 +17,9 @@ message that **names the release line that wrote it** and the exact commands — so you can fetch the right old binary without guessing: ``` -__manifest is stamped at internal schema v8, but this omnigraph reads only v9. -This graph was created by omnigraph 0.12.x. Rebuild it: with an omnigraph -0.12.x binary run `omnigraph export > graph.jsonl`, then with this +__manifest is stamped at internal schema v4, but this omnigraph reads only v9. +This graph was created by omnigraph 0.8.x. Rebuild it: with an omnigraph +0.8.x binary run `omnigraph export > graph.jsonl`, then with this binary run `omnigraph init --schema ` and `omnigraph load --mode overwrite --data graph.jsonl `. (Data, vectors, and blobs are preserved; commit history and branches are not.) See docs/user/operations/upgrade.md. @@ -36,11 +36,15 @@ from that line (the latest is safest): | internal schema v2 | omnigraph 0.4.1–0.6.1 | the latest 0.6.x (e.g. 0.6.1) | | internal schema v3 | omnigraph 0.6.2–0.7.2 | the latest 0.7.x (e.g. 0.7.2) | | internal schema v4 | omnigraph 0.8.x | the latest 0.8.x (e.g. 0.8.1) | -| internal schema v5 | omnigraph 0.9.x | the latest 0.9.x | -| internal schema v6 | omnigraph 0.10.x | the latest 0.10.x | -| internal schema v7 | omnigraph 0.11.x | the latest 0.11.x | -| internal schema v8 | omnigraph 0.12.x | the latest 0.12.x | -| internal schema v9 | omnigraph 0.13.x | — current format; no rebuild needed | +| internal schema v5–v8 | no published release (see below) | a source build at the matching commit | +| internal schema v9 | omnigraph 0.9.x | — current format; no rebuild needed | + +**Stamps v5–v8 never shipped.** The storage format advanced five times inside +the single 0.8.1 → 0.9.0 development window, so the only graphs carrying those +stamps came from source builds off `main`; no published binary reads them and +the refusal message names them `0.9.0-dev`. If you have one, export it with a +build of the commit that created it, then load into a fresh 0.9.x graph. A +released binary only ever wrote v4 (0.8.x) or v9 (0.9.x). You can also check versions before you hit a refusal: @@ -119,11 +123,13 @@ stream-config v3, lifecycle state v2, a manifest-selected recovery-v12's atomic base-plus-token publication. These are physical correctness foundations; v9 does not by itself expose a public streaming API. -A v8 graph must use the standard rebuild recipe above. Quiesce every v8 writer, -export with the latest 0.12.x binary, initialize a **different** root with the -0.13.x binary, load the export, and verify the v9 stamp plus row/vector/blob -fidelity before fleet cutover. Keep the v8 root unchanged through the rollback -window. A v9 binary refuses v8, and a v8 binary refuses v9. +A v8 graph must use the standard rebuild recipe above. Because v8 never +shipped in a release, this affects only source builds off `main` during 0.9.0 +development: quiesce every v8 writer, export with a build of the commit that +created the graph, initialize a **different** root with the 0.9.x binary, load +the export, and verify the v9 stamp plus row/vector/blob fidelity before +cutover. Keep the v8 root unchanged through the rollback window. A v9 binary +refuses v8, and a v8 binary refuses v9. V9's physical attribution field is named `__omnigraph_stream_v1$`. The trailing `$` is deliberately outside the `.pg` property-name grammar, so it cannot diff --git a/openapi.json b/openapi.json index 99df28a5..543a09b3 100644 --- a/openapi.json +++ b/openapi.json @@ -7,7 +7,7 @@ "name": "MIT", "identifier": "MIT" }, - "version": "0.8.1" + "version": "0.9.0" }, "paths": { "/graphs": {