Stop queueing and failing sparse work while hybrid is off

docs/changelog.md

@@ -20,6 +20,18 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/).
   stored with, since the values are an approximation either way; where a
   particular chunk table does need bringing into line, `recreate_chunks()` on
   it rewrites every row through the new path.
+- Upgrading from 1.0 with `pgedge_vectorizer.enable_hybrid` off no longer
+  queues a sparse backfill item for every existing chunk, and the worker no
+  longer raises an error for a sparse-only queue item while hybrid search is
+  off; it completes the item instead, since the chunk already has its dense
+  embedding and there is no sparse one to compute. Previously each such item
+  failed on every attempt until it reached `max_retries`, which on a large
+  database filled the log for days. Items an earlier upgrade has already
+  queued are completed once the new worker reaches them, whilst any that had
+  already run out of attempts stay `failed` and can be removed with
+  `DELETE FROM pgedge_vectorizer.queue WHERE status = 'failed' AND
+  metadata->>'sparse_only' = 'true';`. If hybrid search is enabled later,
+  `reprocess_chunks()` queues the sparse backfill.
 
 ### Added
 

sql/pgedge_vectorizer--1.0--1.1.sql

@@ -1490,7 +1490,13 @@ DECLARE
     src_pk_type TEXT;
     pk_count    INT;
     refreshed   INT;
+    hybrid_on   BOOLEAN;
 BEGIN
+    hybrid_on := COALESCE(
+        current_setting('pgedge_vectorizer.enable_hybrid', true),
+        'false'
+    )::BOOLEAN;
+
     -- Populate vectorizers from existing vectorization triggers
     FOR rec IN
         SELECT c.oid::regclass::text AS source_table,
@@ -1564,12 +1570,17 @@ BEGIN
 
         -- Enqueue existing chunks that have a dense embedding but no sparse
         -- embedding yet so the worker can backfill BM25 scores for them.
-        EXECUTE format(
-            'INSERT INTO pgedge_vectorizer.queue (chunk_id, chunk_table, content, metadata, max_attempts)'
-            ' SELECT id, %L, content, jsonb_build_object(''sparse_only'', true),'
-            '        current_setting(''pgedge_vectorizer.max_retries'')::int FROM %I'
-            ' WHERE embedding IS NOT NULL AND sparse_embedding IS NULL',
-            chk_tbl, chk_tbl);
+        -- Only when hybrid search is on, as reprocess_chunks() does: with it
+        -- off the worker has nothing to compute for these items, and
+        -- reprocess_chunks() backfills them if hybrid is enabled later.
+        IF hybrid_on THEN
+            EXECUTE format(
+                'INSERT INTO pgedge_vectorizer.queue (chunk_id, chunk_table, content, metadata, max_attempts)'
+                ' SELECT id, %L, content, jsonb_build_object(''sparse_only'', true),'
+                '        current_setting(''pgedge_vectorizer.max_retries'')::int FROM %I'
+                ' WHERE embedding IS NOT NULL AND sparse_embedding IS NULL',
+                chk_tbl, chk_tbl);
+        END IF;
     END LOOP;
 
     -- Any vectorizer whose primary key is still unknown, for example a registry

src/worker.c

@@ -1590,6 +1590,7 @@ process_queue_batch(const char *dbname)
 		int batch_count = 0;
 		bool has_retries = false;
 		bool has_sparse_only = false;
+		bool needs_provider = false;
 
 		elog(DEBUG1, "Worker for database \"%s\" processing %d queue items",
 			 dbname, n_items);
@@ -1656,6 +1657,8 @@ process_queue_batch(const char *dbname)
 
 			if (sparse_only[i])
 				has_sparse_only = true;
+			else
+				needs_provider = true;
 		}
 
 		/*
@@ -1688,18 +1691,25 @@ process_queue_batch(const char *dbname)
 				false, 0);
 		}
 
-		/* Get the provider */
-		provider = get_current_provider();
-		if (provider == NULL)
+		/*
+		 * Resolve the provider only when some item needs a dense embedding.
+		 * A pull of sparse-only items never calls it, so a missing or broken
+		 * provider must not fail work that can complete without one.
+		 */
+		if (needs_provider)
 		{
-			elog(ERROR, "No provider configured");
-		}
+			provider = get_current_provider();
+			if (provider == NULL)
+			{
+				elog(ERROR, "No provider configured");
+			}
 
-		/* Initialize provider if needed */
-		if (!provider->init(&error_msg))
-		{
-			elog(ERROR, "Failed to initialize provider: %s",
-				 error_msg ? error_msg : "unknown error");
+			/* Initialize provider if needed */
+			if (!provider->init(&error_msg))
+			{
+				elog(ERROR, "Failed to initialize provider: %s",
+					 error_msg ? error_msg : "unknown error");
+			}
 		}
 
 		/* Process items in requests as large as batch_extent() allows */
@@ -1823,7 +1833,20 @@ process_queue_batch(const char *dbname)
 					if (!sparse_only[idx])
 						update_embedding(chunk_ids[idx], chunk_tables[idx], embeddings[i], dim);
 					else if (!pgedge_vectorizer_enable_hybrid)
-						elog(ERROR, "cannot process sparse-only queue item while pgedge_vectorizer.enable_hybrid is disabled");
+					{
+						/*
+						 * The dense embedding is already in place and there
+						 * is no sparse one to compute, so the item has
+						 * nothing left to do and is completed as it stands.
+						 * Raising instead would fail it on every attempt,
+						 * filling the log until max_attempts is reached.
+						 * reprocess_chunks() queues the sparse work again if
+						 * hybrid search is enabled later.
+						 */
+						elog(DEBUG1, "Worker for database \"%s\": skipping sparse-only "
+							 "queue item %ld because pgedge_vectorizer.enable_hybrid "
+							 "is disabled", dbname, queue_ids[idx]);
+					}
 
 					/*
 					 * BM25 sparse vector update (opt-in via

test/t/011_sparse_only_without_hybrid.pl

@@ -0,0 +1,123 @@
+# Copyright (c) 2025 - 2026, pgEdge, Inc.
+#
+# Verify that sparse-only work is neither queued nor failed while
+# pgedge_vectorizer.enable_hybrid is off.
+#
+# The 1.0 to 1.1 upgrade used to queue a sparse_only backfill item for every
+# existing chunk regardless of the setting, and the worker raised an ERROR for
+# each of them, so a database with many chunks filled its log for days before
+# every item ran out of attempts. The upgrade is exercised from a real 1.0
+# install, once with hybrid off and once with it on, and the worker is then
+# handed a sparse_only item with hybrid off and must complete it quietly.
+#
+# The provider is set to openai with no API key file, so it cannot be
+# initialised. A sparse-only item needs no provider, so that must not stop the
+# worker completing one.
+
+use strict;
+use warnings;
+
+# See the comment in 001_worker_coverage.pl about loading these at compile time.
+use PostgreSQL::Test::Cluster;
+use PostgreSQL::Test::Utils;
+use Test::More;
+
+my $node = PostgreSQL::Test::Cluster->new('vectorizer_sparse_no_hybrid');
+$node->init;
+$node->append_conf(
+	'postgresql.conf', qq(
+shared_preload_libraries = 'pgedge_vectorizer'
+pgedge_vectorizer.worker_poll_interval = 200
+pgedge_vectorizer.batch_size = 1
+pgedge_vectorizer.provider = 'openai'
+pgedge_vectorizer.enable_hybrid = false
+max_worker_processes = 16
+));
+
+$node->start;
+
+# Build a 1.0 vectorizer with embedded chunks, then upgrade it to 1.1 with
+# enable_hybrid set as given, and return the number of sparse_only items the
+# upgrade queued. No worker serves these databases, so nothing is consumed.
+sub upgraded_sparse_items
+{
+	my ($dbname, $hybrid) = @_;
+
+	$node->safe_psql('postgres', "CREATE DATABASE $dbname");
+	$node->safe_psql($dbname, 'CREATE EXTENSION vector');
+	$node->safe_psql($dbname,
+		"CREATE EXTENSION pgedge_vectorizer VERSION '1.0'");
+	$node->safe_psql(
+		$dbname, q(
+CREATE TABLE docs (id INT PRIMARY KEY, body TEXT);
+SELECT pgedge_vectorizer.enable_vectorization('docs', 'body',
+                                              embedding_dimension => 3);
+INSERT INTO docs VALUES (1, 'alpha beta'), (2, 'gamma delta');
+UPDATE docs_body_chunks SET embedding = '[1,2,3]';
+DELETE FROM pgedge_vectorizer.queue;
+));
+
+	return $node->safe_psql(
+		$dbname, qq(
+SET pgedge_vectorizer.enable_hybrid = $hybrid;
+ALTER EXTENSION pgedge_vectorizer UPDATE TO '1.1';
+SELECT count(*) FROM pgedge_vectorizer.queue
+WHERE (metadata->>'sparse_only')::boolean;
+));
+}
+
+is(upgraded_sparse_items('upgrade_hybrid_off', 'off'),
+	'0', 'the 1.0 to 1.1 upgrade queues no sparse backfill with hybrid off');
+
+is(upgraded_sparse_items('upgrade_hybrid_on', 'on'),
+	'2', 'the 1.0 to 1.1 upgrade still queues the sparse backfill with hybrid on');
+
+# Now the worker: an item already queued, for example by an earlier upgrade,
+# must be completed rather than failed while hybrid is off.
+my $dbname = 'worker_hybrid_off';
+
+$node->safe_psql('postgres', "CREATE DATABASE $dbname");
+$node->safe_psql($dbname, 'CREATE EXTENSION vector');
+$node->safe_psql($dbname, 'CREATE EXTENSION pgedge_vectorizer');
+$node->safe_psql(
+	$dbname, q(
+CREATE TABLE chunks (
+    id               BIGSERIAL PRIMARY KEY,
+    source_id        BIGINT,
+    chunk_index      INT,
+    content          TEXT,
+    token_count      INT,
+    embedding        vector(3),
+    sparse_embedding sparsevec(65536)
+);
+INSERT INTO chunks (source_id, chunk_index, content, token_count, embedding)
+VALUES (1, 0, 'alpha', 1, '[1,2,3]');
+INSERT INTO pgedge_vectorizer.queue (chunk_id, chunk_table, content, status, metadata)
+VALUES (1, 'chunks', 'alpha', 'pending', '{"sparse_only": true}'::jsonb);
+));
+
+$node->append_conf('postgresql.conf',
+	"pgedge_vectorizer.databases = '$dbname'\n");
+$node->reload;
+
+ok( $node->poll_query_until(
+		$dbname,
+		"SELECT status = 'completed' OR attempts > 0 FROM pgedge_vectorizer.queue WHERE chunk_id = 1"
+	),
+	'the worker picks up the sparse-only item');
+
+is( $node->safe_psql(
+		$dbname,
+		"SELECT status || ':' || attempts || ':' || coalesce(error_message, '') "
+		  . "FROM pgedge_vectorizer.queue WHERE chunk_id = 1"),
+	'completed:0:',
+	'a sparse-only item is completed, not failed, while hybrid is off');
+
+is( $node->safe_psql(
+		$dbname,
+		'SELECT sparse_embedding IS NULL AND embedding IS NOT NULL FROM chunks WHERE id = 1'),
+	't',
+	'the chunk keeps its dense embedding and gains no sparse one');
+
+$node->stop;
+done_testing();