From 99aec8516da2158b150687d75a54549bb5c27e21 Mon Sep 17 00:00:00 2001 From: Dave Page Date: Thu, 1 Oct 2026 11:47:55 +0100 Subject: [PATCH 1/2] Stop queueing and failing sparse work while hybrid is off The 1.0 to 1.1 upgrade queued a sparse_only backfill item for every existing chunk whether or not pgedge_vectorizer.enable_hybrid was on, and the worker raised an ERROR for each of them, so every item failed on every attempt until it reached max_retries. On a database with tens of thousands of chunks that filled the log for days. The backfill now checks enable_hybrid, as reprocess_chunks() already does, and reprocess_chunks() queues the work if hybrid is enabled later. The worker completes a sparse-only item while hybrid is off rather than raising, since its dense embedding is already in place and there is nothing else for it to do, which also clears items an earlier upgrade has already queued. Closes #83 --- docs/changelog.md | 12 +++ sql/pgedge_vectorizer--1.0--1.1.sql | 23 +++-- src/worker.c | 15 ++- test/t/011_sparse_only_without_hybrid.pl | 122 +++++++++++++++++++++++ 4 files changed, 165 insertions(+), 7 deletions(-) create mode 100644 test/t/011_sparse_only_without_hybrid.pl diff --git a/docs/changelog.md b/docs/changelog.md index a92bff1..242bba9 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -20,6 +20,18 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/). stored with, since the values are an approximation either way; where a particular chunk table does need bringing into line, `recreate_chunks()` on it rewrites every row through the new path. +- Upgrading from 1.0 with `pgedge_vectorizer.enable_hybrid` off no longer + queues a sparse backfill item for every existing chunk, and the worker no + longer raises an error for a sparse-only queue item while hybrid search is + off; it completes the item instead, since the chunk already has its dense + embedding and there is no sparse one to compute. Previously each such item + failed on every attempt until it reached `max_retries`, which on a large + database filled the log for days. Items an earlier upgrade has already + queued are completed once the new worker reaches them, whilst any that had + already run out of attempts stay `failed` and can be removed with + `DELETE FROM pgedge_vectorizer.queue WHERE status = 'failed' AND + metadata->>'sparse_only' = 'true';`. If hybrid search is enabled later, + `reprocess_chunks()` queues the sparse backfill. ### Added diff --git a/sql/pgedge_vectorizer--1.0--1.1.sql b/sql/pgedge_vectorizer--1.0--1.1.sql index 6bc08c0..deb96c1 100644 --- a/sql/pgedge_vectorizer--1.0--1.1.sql +++ b/sql/pgedge_vectorizer--1.0--1.1.sql @@ -1490,7 +1490,13 @@ DECLARE src_pk_type TEXT; pk_count INT; refreshed INT; + hybrid_on BOOLEAN; BEGIN + hybrid_on := COALESCE( + current_setting('pgedge_vectorizer.enable_hybrid', true), + 'false' + )::BOOLEAN; + -- Populate vectorizers from existing vectorization triggers FOR rec IN SELECT c.oid::regclass::text AS source_table, @@ -1564,12 +1570,17 @@ BEGIN -- Enqueue existing chunks that have a dense embedding but no sparse -- embedding yet so the worker can backfill BM25 scores for them. - EXECUTE format( - 'INSERT INTO pgedge_vectorizer.queue (chunk_id, chunk_table, content, metadata, max_attempts)' - ' SELECT id, %L, content, jsonb_build_object(''sparse_only'', true),' - ' current_setting(''pgedge_vectorizer.max_retries'')::int FROM %I' - ' WHERE embedding IS NOT NULL AND sparse_embedding IS NULL', - chk_tbl, chk_tbl); + -- Only when hybrid search is on, as reprocess_chunks() does: with it + -- off the worker has nothing to compute for these items, and + -- reprocess_chunks() backfills them if hybrid is enabled later. + IF hybrid_on THEN + EXECUTE format( + 'INSERT INTO pgedge_vectorizer.queue (chunk_id, chunk_table, content, metadata, max_attempts)' + ' SELECT id, %L, content, jsonb_build_object(''sparse_only'', true),' + ' current_setting(''pgedge_vectorizer.max_retries'')::int FROM %I' + ' WHERE embedding IS NOT NULL AND sparse_embedding IS NULL', + chk_tbl, chk_tbl); + END IF; END LOOP; -- Any vectorizer whose primary key is still unknown, for example a registry diff --git a/src/worker.c b/src/worker.c index f1f11d9..b90ed72 100644 --- a/src/worker.c +++ b/src/worker.c @@ -1823,7 +1823,20 @@ process_queue_batch(const char *dbname) if (!sparse_only[idx]) update_embedding(chunk_ids[idx], chunk_tables[idx], embeddings[i], dim); else if (!pgedge_vectorizer_enable_hybrid) - elog(ERROR, "cannot process sparse-only queue item while pgedge_vectorizer.enable_hybrid is disabled"); + { + /* + * The dense embedding is already in place and there + * is no sparse one to compute, so the item has + * nothing left to do and is completed as it stands. + * Raising instead would fail it on every attempt, + * filling the log until max_attempts is reached. + * reprocess_chunks() queues the sparse work again if + * hybrid search is enabled later. + */ + elog(DEBUG1, "Worker for database \"%s\": skipping sparse-only " + "queue item %ld because pgedge_vectorizer.enable_hybrid " + "is disabled", dbname, queue_ids[idx]); + } /* * BM25 sparse vector update (opt-in via diff --git a/test/t/011_sparse_only_without_hybrid.pl b/test/t/011_sparse_only_without_hybrid.pl new file mode 100644 index 0000000..1373d75 --- /dev/null +++ b/test/t/011_sparse_only_without_hybrid.pl @@ -0,0 +1,122 @@ +# Copyright (c) 2025 - 2026, pgEdge, Inc. +# +# Verify that sparse-only work is neither queued nor failed while +# pgedge_vectorizer.enable_hybrid is off. +# +# The 1.0 to 1.1 upgrade used to queue a sparse_only backfill item for every +# existing chunk regardless of the setting, and the worker raised an ERROR for +# each of them, so a database with many chunks filled its log for days before +# every item ran out of attempts. The upgrade is exercised from a real 1.0 +# install, once with hybrid off and once with it on, and the worker is then +# handed a sparse_only item with hybrid off and must complete it quietly. +# +# The provider is set to ollama, the one provider whose init needs no API key, +# because the worker resolves and initialises it before reaching the item. + +use strict; +use warnings; + +# See the comment in 001_worker_coverage.pl about loading these at compile time. +use PostgreSQL::Test::Cluster; +use PostgreSQL::Test::Utils; +use Test::More; + +my $node = PostgreSQL::Test::Cluster->new('vectorizer_sparse_no_hybrid'); +$node->init; +$node->append_conf( + 'postgresql.conf', qq( +shared_preload_libraries = 'pgedge_vectorizer' +pgedge_vectorizer.worker_poll_interval = 200 +pgedge_vectorizer.batch_size = 1 +pgedge_vectorizer.provider = 'ollama' +pgedge_vectorizer.enable_hybrid = false +max_worker_processes = 16 +)); + +$node->start; + +# Build a 1.0 vectorizer with embedded chunks, then upgrade it to 1.1 with +# enable_hybrid set as given, and return the number of sparse_only items the +# upgrade queued. No worker serves these databases, so nothing is consumed. +sub upgraded_sparse_items +{ + my ($dbname, $hybrid) = @_; + + $node->safe_psql('postgres', "CREATE DATABASE $dbname"); + $node->safe_psql($dbname, 'CREATE EXTENSION vector'); + $node->safe_psql($dbname, + "CREATE EXTENSION pgedge_vectorizer VERSION '1.0'"); + $node->safe_psql( + $dbname, q( +CREATE TABLE docs (id INT PRIMARY KEY, body TEXT); +SELECT pgedge_vectorizer.enable_vectorization('docs', 'body', + embedding_dimension => 3); +INSERT INTO docs VALUES (1, 'alpha beta'), (2, 'gamma delta'); +UPDATE docs_body_chunks SET embedding = '[1,2,3]'; +DELETE FROM pgedge_vectorizer.queue; +)); + + return $node->safe_psql( + $dbname, qq( +SET pgedge_vectorizer.enable_hybrid = $hybrid; +ALTER EXTENSION pgedge_vectorizer UPDATE TO '1.1'; +SELECT count(*) FROM pgedge_vectorizer.queue +WHERE (metadata->>'sparse_only')::boolean; +)); +} + +is(upgraded_sparse_items('upgrade_hybrid_off', 'off'), + '0', 'the 1.0 to 1.1 upgrade queues no sparse backfill with hybrid off'); + +is(upgraded_sparse_items('upgrade_hybrid_on', 'on'), + '2', 'the 1.0 to 1.1 upgrade still queues the sparse backfill with hybrid on'); + +# Now the worker: an item already queued, for example by an earlier upgrade, +# must be completed rather than failed while hybrid is off. +my $dbname = 'worker_hybrid_off'; + +$node->safe_psql('postgres', "CREATE DATABASE $dbname"); +$node->safe_psql($dbname, 'CREATE EXTENSION vector'); +$node->safe_psql($dbname, 'CREATE EXTENSION pgedge_vectorizer'); +$node->safe_psql( + $dbname, q( +CREATE TABLE chunks ( + id BIGSERIAL PRIMARY KEY, + source_id BIGINT, + chunk_index INT, + content TEXT, + token_count INT, + embedding vector(3), + sparse_embedding sparsevec(65536) +); +INSERT INTO chunks (source_id, chunk_index, content, token_count, embedding) +VALUES (1, 0, 'alpha', 1, '[1,2,3]'); +INSERT INTO pgedge_vectorizer.queue (chunk_id, chunk_table, content, status, metadata) +VALUES (1, 'chunks', 'alpha', 'pending', '{"sparse_only": true}'::jsonb); +)); + +$node->append_conf('postgresql.conf', + "pgedge_vectorizer.databases = '$dbname'\n"); +$node->reload; + +ok( $node->poll_query_until( + $dbname, + "SELECT status = 'completed' OR attempts > 0 FROM pgedge_vectorizer.queue WHERE chunk_id = 1" + ), + 'the worker picks up the sparse-only item'); + +is( $node->safe_psql( + $dbname, + "SELECT status || ':' || attempts || ':' || coalesce(error_message, '') " + . "FROM pgedge_vectorizer.queue WHERE chunk_id = 1"), + 'completed:0:', + 'a sparse-only item is completed, not failed, while hybrid is off'); + +is( $node->safe_psql( + $dbname, + 'SELECT sparse_embedding IS NULL AND embedding IS NOT NULL FROM chunks WHERE id = 1'), + 't', + 'the chunk keeps its dense embedding and gains no sparse one'); + +$node->stop; +done_testing(); From 3f7fd233205d27da15c649af44d17f903609a2b1 Mon Sep 17 00:00:00 2001 From: Dave Page Date: Thu, 1 Oct 2026 11:58:53 +0100 Subject: [PATCH 2/2] Resolve the provider only when a pull needs a dense embedding A pull made up entirely of sparse-only items never calls the provider, so a missing or unusable one should not fail it. --- src/worker.c | 30 ++++++++++++++++-------- test/t/011_sparse_only_without_hybrid.pl | 7 +++--- 2 files changed, 24 insertions(+), 13 deletions(-) diff --git a/src/worker.c b/src/worker.c index b90ed72..5071a71 100644 --- a/src/worker.c +++ b/src/worker.c @@ -1590,6 +1590,7 @@ process_queue_batch(const char *dbname) int batch_count = 0; bool has_retries = false; bool has_sparse_only = false; + bool needs_provider = false; elog(DEBUG1, "Worker for database \"%s\" processing %d queue items", dbname, n_items); @@ -1656,6 +1657,8 @@ process_queue_batch(const char *dbname) if (sparse_only[i]) has_sparse_only = true; + else + needs_provider = true; } /* @@ -1688,18 +1691,25 @@ process_queue_batch(const char *dbname) false, 0); } - /* Get the provider */ - provider = get_current_provider(); - if (provider == NULL) + /* + * Resolve the provider only when some item needs a dense embedding. + * A pull of sparse-only items never calls it, so a missing or broken + * provider must not fail work that can complete without one. + */ + if (needs_provider) { - elog(ERROR, "No provider configured"); - } + provider = get_current_provider(); + if (provider == NULL) + { + elog(ERROR, "No provider configured"); + } - /* Initialize provider if needed */ - if (!provider->init(&error_msg)) - { - elog(ERROR, "Failed to initialize provider: %s", - error_msg ? error_msg : "unknown error"); + /* Initialize provider if needed */ + if (!provider->init(&error_msg)) + { + elog(ERROR, "Failed to initialize provider: %s", + error_msg ? error_msg : "unknown error"); + } } /* Process items in requests as large as batch_extent() allows */ diff --git a/test/t/011_sparse_only_without_hybrid.pl b/test/t/011_sparse_only_without_hybrid.pl index 1373d75..46873c3 100644 --- a/test/t/011_sparse_only_without_hybrid.pl +++ b/test/t/011_sparse_only_without_hybrid.pl @@ -10,8 +10,9 @@ # install, once with hybrid off and once with it on, and the worker is then # handed a sparse_only item with hybrid off and must complete it quietly. # -# The provider is set to ollama, the one provider whose init needs no API key, -# because the worker resolves and initialises it before reaching the item. +# The provider is set to openai with no API key file, so it cannot be +# initialised. A sparse-only item needs no provider, so that must not stop the +# worker completing one. use strict; use warnings; @@ -28,7 +29,7 @@ shared_preload_libraries = 'pgedge_vectorizer' pgedge_vectorizer.worker_poll_interval = 200 pgedge_vectorizer.batch_size = 1 -pgedge_vectorizer.provider = 'ollama' +pgedge_vectorizer.provider = 'openai' pgedge_vectorizer.enable_hybrid = false max_worker_processes = 16 ));