Remove Parquet code

.github/workflows/ci.yml

@@ -68,10 +68,10 @@ jobs:
           fi
 
       - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
-      - name: Install Ubuntu packages
-        run: |
-          sudo apt-get update
-          sudo apt-get -y install protobuf-compiler
+      - name: Install protoc
+        uses: taiki-e/install-action@7769b73c2ec98c38dfcf2e18c83cfd4880c038c1
+        with:
+          tool: protoc
       - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v.6.2.0
         with:
           python-version: '3.11'
@@ -101,17 +101,17 @@ jobs:
         uses: taiki-e/install-action@7769b73c2ec98c38dfcf2e18c83cfd4880c038c1
         with:
           tool: cargo-nextest
-      - name: cargo check without metrics
+      - name: cargo check
         if: always() && steps.modified.outputs.rust_src == 'true'
         run: cargo check --features=postgres --tests --bin quickwit
         working-directory: ./quickwit
-      - name: cargo check with metrics
+      - name: cargo check with DataFusion
         if: always() && steps.modified.outputs.rust_src == 'true'
-        run: cargo check --features=postgres,metrics --tests --bin quickwit
+        run: cargo check --features=postgres,datafusion --tests --bin quickwit
         working-directory: ./quickwit
       - name: cargo nextest
         if: always() && steps.modified.outputs.rust_src == 'true'
-        run: cargo nextest run --features=postgres,metrics --retries 1
+        run: cargo nextest run --features=postgres --retries 1
         working-directory: ./quickwit
       - name: Install python packages
         if: always() && steps.modified.outputs.rust_src == 'true'

.github/workflows/datafusion-ci.yml

@@ -45,11 +45,11 @@ jobs:
               - quickwit/**/*.toml
               - quickwit/**/*.proto
               - .github/workflows/datafusion-ci.yml
-      - name: Install Ubuntu packages
+      - name: Install protoc
         if: steps.modified.outputs.rust_src == 'true'
-        run: |
-          sudo apt-get update
-          sudo apt-get -y install protobuf-compiler
+        uses: taiki-e/install-action@7769b73c2ec98c38dfcf2e18c83cfd4880c038c1
+        with:
+          tool: protoc
       - name: Setup stable Rust Toolchain
         if: steps.modified.outputs.rust_src == 'true'
         uses: dtolnay/rust-toolchain@3c5f7ea28cd621ae0bf5283f0e981fb97b8a7af9 # master
@@ -61,11 +61,11 @@ jobs:
         with:
           workspaces: "./quickwit -> target"
           shared-key: "quickwit-cargo"
-      - name: cargo test quickwit-datafusion
+      - name: cargo test DataFusion crates
         if: always() && steps.modified.outputs.rust_src == 'true'
-        run: cargo test -p quickwit-datafusion --tests
+        run: cargo test -p quickwit-df-core -p quickwit-datafusion --all-features
         working-directory: ./quickwit
-      - name: cargo test quickwit-integration-tests with datafusion
+      - name: cargo check server with DataFusion
         if: always() && steps.modified.outputs.rust_src == 'true'
-        run: cargo test -p quickwit-integration-tests --features datafusion metrics_distributed_tests -- --test-threads=1
+        run: cargo check -p quickwit-cli --features quickwit-metastore/postgres,datafusion --bin quickwit
         working-directory: ./quickwit

.github/workflows/dependency.yml

@@ -20,11 +20,4 @@ jobs:
         with:
           # GHSA-c38w-74pg-36hr, GHSA-4grx-2x9w-596c: minor vuln on the rsa crate, used for google storage.
           # GHSA-cq8v-f236-94qc: rand 0.8.6 unsound with custom logger + rand::rng(), not affected (log feature disabled, transitive dep from fail/sqlx).
-          # GHSA-2f9f-gq7v-9h6m: thrift 0.17 excessive-allocation on untrusted input. Already a transitive
-          #   dep via parquet on main; PR-4 (streaming Parquet reader) adds it as a direct dep for
-          #   `parquet::format::PageHeader::read_from_in_protocol`. Inputs are parquet files we wrote to
-          #   our own object store (trusted source). Defense-in-depth: the streaming reader caps each
-          #   Thrift parse buffer at `max_page_header_bytes` (1 MiB default) BEFORE handing bytes to
-          #   thrift, so the outer buffer thrift can read from is bounded regardless of any inner
-          #   length prefix. No newer thrift release fixes this — 0.17.0 is the latest on crates.io.
           allow-ghsas: GHSA-c38w-74pg-36hr,GHSA-4grx-2x9w-596c,GHSA-cq8v-f236-94qc,GHSA-2f9f-gq7v-9h6m

CHANGELOG.md

@@ -37,9 +37,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 ### Added
 - Add Ingest V2 — now the default (#5600, #5566, #5463, #5375, #5350, #5252, #5202, #6078, #6185, #6203, #6207, #6217, #6249)
 - Offload leaf-search work to AWS Lambda functions — searchers can farm out part of their workload to Lambda (#6157, #5c1e60f)
-- Add experimental DataFusion query layer for Parquet metrics with SQL/Substrait execution, Arrow IPC streaming, and distributed workers (#6276)
-- Extend the Parquet metrics pipeline with DDSketch, sorted-series and row-key metadata, zone-map pruning, partitioning, sketch split pagination, and Substrait execution metadata (#6248, #6257, #6290, #6292, #6295, #6340, #6347, #6348, #6349, #6363, #6364, #6368)
-- Add column-major and streaming Parquet merge primitives, merge scheduler wiring, and a rollout flag for routing regular merges through the streaming engine (#6335, #6362, #6367, #6377, #6384, #6386, #6406, #6407, #6408, #6409, #6423, #6424, #6425, #6426, #6428, #6441)
+- Add experimental DataFusion query support with SQL/Substrait execution, Arrow IPC streaming, and distributed workers (#6276)
 - Add optional mTLS validation to the REST API and an optional dedicated health-check HTTP server for deployments that put the REST API behind mTLS (#6467, #6528)
 - Add SQS source (#5374, #5335, #5148)
 - Add Jaeger v2 support (#6023)
@@ -153,8 +151,6 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - Fix bug when deploying a new routing table (#6249)
 - Fix retiring/decommissioned indexer handling, shard ownership deletion, empty-shard indexing checks, and material indexer pool updates (#6427, #6504, #6509, #6518, #6553)
 - Fix control-plane scheduling idempotency and unavailable-node reporting (#6503, #6524, #6551)
-- Fix descending sorted-series encoding, null ordering in sorted-series keys, and maturity assignment on Parquet splits (#6338, #6343, #6399)
-- Fix Parquet merge adapter and consumer edge cases around sorted input, sub-regions, and stronger verifiers (#6426, #6428)
 - Fix OpenDAL GCS TLS regression, GCS put memory growth, single-part MD5 headers, S3 stalled-stream retryability, S3 dispatch retries, and localstack checksum compatibility (#6478, #6482, #6485, #6492, #6501, #6502, #6539, #6541)
 - Fix Azure delete metrics for the Azure blob backend (#6469)
 - Fix Lambda retry log storms and empty `LAMBDA_ZIP_PATH` handling (#6318, #6462)