Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 12 additions & 2 deletions src/pg-copy/copy-retry-batch.lisp
Original file line number Diff line number Diff line change
Expand Up @@ -194,8 +194,18 @@
(parse-copy-error-context
(database-error-context next-error-in-batch))))

(setf condition next-error-in-batch
next-error (+ current-batch-pos next-error-relative)))))))))
(if next-error-relative
(setf condition next-error-in-batch
next-error (+ current-batch-pos next-error-relative))
;; No COPY line-number context (e.g. FK violation following
;; a format error): fall back to binary search for the
;; remaining rows and exit.
(progn
(incf nb-errors
(bisect-batch table-name table columns
batch current-batch-pos
(- (batch-count batch) current-batch-pos)))
(return-from retry-batch nb-errors))))))))))

(log-message :info "Recovery found ~d errors in ~d row~:p"
nb-errors (batch-count batch))
Expand Down
1 change: 1 addition & 0 deletions test/Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@ REGRESS= allcols.load \
copy-hex.load \
dbf.load \
errors.load \
fk-reject.load \
fixed.load \
fields-with-periods.load \
ixf.load \
Expand Down
41 changes: 41 additions & 0 deletions test/fk-reject.load
Original file line number Diff line number Diff line change
@@ -0,0 +1,41 @@
/*
* Regression test for GitHub issue #1333.
*
* When a batch contains rows that violate a foreign-key constraint, pgloader
* must reject those rows (write them to the .dat reject file) and commit the
* valid rows. Previously the retry-batch code crashed with "Invalid index N
* for (SIMPLE-VECTOR N)" instead, and the rejected rows were lost entirely.
*
* The fix uses binary search (bisect-batch) when PostgreSQL does not annotate
* the COPY error with a line number — which is the case for FK violations.
*
* Scenario:
* parent table: ids 1, 2, 3
* child table: FK → parent(id)
* CSV rows: (1,1) valid | (2,99) bad FK | (3,2) valid
*
* Expected: rows (1,1) and (3,2) are loaded; (2,99) is rejected.
*/

LOAD CSV
FROM INLINE (id, parent_id)
INTO postgresql:///pgloader?fk_reject

WITH truncate,
fields terminated by ','

SET work_mem to '12MB'

BEFORE LOAD DO
$$ drop table if exists fk_reject cascade; $$,
$$ drop table if exists fk_parent cascade; $$,
$$ create table fk_parent (id integer primary key); $$,
$$ insert into fk_parent values (1), (2), (3); $$,
$$ create table fk_reject (
id integer primary key,
parent_id integer not null references fk_parent(id)
); $$;

1,1
2,99
3,2
2 changes: 2 additions & 0 deletions test/regress/expected/fk-reject.out
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
1 1
3 2
Loading