From 04c1be5b2c8b4044a0f95150a80f8c56fa0771fa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Grzegorz=20=C5=9Aliwi=C5=84ski?= Date: Sat, 25 Jul 2026 16:08:48 +0200 Subject: [PATCH 1/4] Pass autocomit to the janitor's load methods - closes #902 --- README.rst | 19 ++++++++++ newsfragments/902.feature.rst | 1 + pytest_postgresql/config.py | 2 + pytest_postgresql/factories/noprocess.py | 5 +++ pytest_postgresql/factories/process.py | 5 +++ pytest_postgresql/janitor.py | 9 +++++ pytest_postgresql/plugin.py | 13 +++++++ .../test_assert_load_autocommit_is_true.py | 14 +++++++ tests/test_janitor.py | 37 +++++++++++++++++++ tests/test_postgres_options_plugin.py | 15 ++++++++ 10 files changed, 120 insertions(+) create mode 100644 newsfragments/902.feature.rst create mode 100644 tests/examples/test_assert_load_autocommit_is_true.py diff --git a/README.rst b/README.rst index 6d621c2a..8122a964 100644 --- a/README.rst +++ b/README.rst @@ -244,6 +244,19 @@ Defining pre-population on the command line: pytest --postgresql-load=path/to/file.sql --postgresql-load=path.to.function +If a loaded ``.sql`` file contains statements that cannot run inside a transaction +block (for example ``CREATE DATABASE``), enable autocommit on the loader connection. +This can be set without writing a custom loader, via the factory argument, the command +line, or ``pytest.ini``: + +.. code-block:: python + + postgresql_my_proc = factories.postgresql_proc(load=[Path("with_create_db.sql")], load_autocommit=True) + +.. code-block:: sh + + pytest --postgresql-load-autocommit + Connecting to an existing PostgreSQL database ---------------------------------------------- @@ -370,6 +383,12 @@ You can define settings via fixture factory arguments, command line options, or - postgresql_load - yes - + * - Autocommit for the SQL loader connection + - load_autocommit + - --postgresql-load-autocommit + - postgresql_load_autocommit + - - + - False * - PostgreSQL connection options - options - --postgresql-options diff --git a/newsfragments/902.feature.rst b/newsfragments/902.feature.rst new file mode 100644 index 00000000..61421d99 --- /dev/null +++ b/newsfragments/902.feature.rst @@ -0,0 +1 @@ +Allow running the SQL loader connection with autocommit enabled, so ``.sql`` files containing statements that cannot run inside a transaction block (such as ``CREATE DATABASE``) can be loaded. This is configurable without a custom loader in three ways: the ``postgresql_load_autocommit`` ini option, the ``--postgresql-load-autocommit`` command line flag, or the ``load_autocommit`` argument of the ``postgresql_proc`` / ``postgresql_noproc`` factories (which also maps to an ``autocommit`` argument on ``DatabaseJanitor`` / ``AsyncDatabaseJanitor``). diff --git a/pytest_postgresql/config.py b/pytest_postgresql/config.py index 962c7de8..cccfc5b3 100644 --- a/pytest_postgresql/config.py +++ b/pytest_postgresql/config.py @@ -23,6 +23,7 @@ class PostgreSQLConfig: unixsocketdir: str dbname: str load: list[Path | str] + load_autocommit: bool postgres_options: str drop_test_database: bool @@ -49,6 +50,7 @@ def get_postgresql_option(option: str) -> Any: unixsocketdir=get_postgresql_option("unixsocketdir"), dbname=get_postgresql_option("dbname"), load=load_paths, + load_autocommit=bool(get_postgresql_option("load_autocommit")), postgres_options=get_postgresql_option("postgres_options"), drop_test_database=request.config.getoption("postgresql_drop_test_database"), ) diff --git a/pytest_postgresql/factories/noprocess.py b/pytest_postgresql/factories/noprocess.py index 2d7f8b49..da5fbd40 100644 --- a/pytest_postgresql/factories/noprocess.py +++ b/pytest_postgresql/factories/noprocess.py @@ -45,6 +45,7 @@ def postgresql_noproc( dbname: str | None = None, options: str = "", load: list[Callable | str | Path] | None = None, + load_autocommit: bool = False, depends_on: str | None = None, ) -> Callable[[FixtureRequest], Iterator[NoopExecutor]]: """Postgresql noprocess factory. @@ -56,6 +57,9 @@ def postgresql_noproc( :param dbname: postgresql database name :param options: Postgresql connection options :param load: List of functions used to initialize database's template. + :param load_autocommit: run the SQL loader connection with autocommit on. + Required for statements that cannot run inside a transaction block, + e.g. ``CREATE DATABASE`` in a loaded ``.sql`` file. :param depends_on: Optional name of the fixture to depend on. :returns: function which makes a postgresql process """ @@ -113,6 +117,7 @@ def postgresql_noproc_fixture(request: FixtureRequest) -> Iterator[NoopExecutor] as_template=True, version=noop_exec.version, password=noop_exec.password, + autocommit=load_autocommit or config.load_autocommit, ) if drop_test_database: janitor.drop() diff --git a/pytest_postgresql/factories/process.py b/pytest_postgresql/factories/process.py index d4f80d86..c0376429 100644 --- a/pytest_postgresql/factories/process.py +++ b/pytest_postgresql/factories/process.py @@ -94,6 +94,7 @@ def postgresql_proc( unixsocketdir: str | None = None, postgres_options: str | None = None, load: list[Callable | str | Path] | None = None, + load_autocommit: bool = False, ) -> Callable[[FixtureRequest, TempPathFactory], PostgreSQLExecutor]: """Postgresql process factory. @@ -114,6 +115,9 @@ def postgresql_proc( :param unixsocketdir: directory to create postgresql's unixsockets :param postgres_options: Postgres executable options for use by pg_ctl :param load: List of functions used to initialize database's template. + :param load_autocommit: run the SQL loader connection with autocommit on. + Required for statements that cannot run inside a transaction block, + e.g. ``CREATE DATABASE`` in a loaded ``.sql`` file. :returns: function which makes a postgresql process """ @@ -220,6 +224,7 @@ def _cleanup_executor_resources() -> None: as_template=True, version=postgresql_executor.version, password=postgresql_executor.password, + autocommit=load_autocommit or config.load_autocommit, ) if config.drop_test_database: janitor.drop() diff --git a/pytest_postgresql/janitor.py b/pytest_postgresql/janitor.py index babf631b..f40ae7c8 100644 --- a/pytest_postgresql/janitor.py +++ b/pytest_postgresql/janitor.py @@ -34,6 +34,7 @@ class BaseDatabaseJanitor: as_template: bool _connection_timeout: int isolation_level: "psycopg.IsolationLevel | None" + autocommit: bool version: Version # type: ignore[valid-type] def __init__( @@ -48,6 +49,7 @@ def __init__( as_template: bool = False, password: str | None = None, isolation_level: "psycopg.IsolationLevel | None" = None, + autocommit: bool = False, connection_timeout: int = 60, ) -> None: """Initialize janitor. @@ -62,6 +64,10 @@ def __init__( :param password: optional postgresql password :param isolation_level: optional postgresql isolation level defaults to server's default + :param autocommit: run the SQL loader connection with autocommit on. + Required for statements that cannot run inside a transaction + block, e.g. ``CREATE DATABASE`` in a loaded ``.sql`` file. Only + affects the loader connection, not the janitor's admin cursor. :param connection_timeout: how long to retry connection before raising a TimeoutError """ @@ -74,6 +80,7 @@ def __init__( self.as_template = as_template self._connection_timeout = connection_timeout self.isolation_level = isolation_level + self.autocommit = autocommit if not isinstance(version, Version): self.version = parse(str(version)) else: @@ -154,6 +161,7 @@ def load(self, load: Callable | str | Path) -> None: user=self.user, dbname=self.dbname, password=self.password, + autocommit=self.autocommit, ) @contextmanager @@ -259,6 +267,7 @@ async def load(self, load: Callable | str | Path) -> None: "user": self.user, "dbname": self.dbname, "password": self.password, + "autocommit": self.autocommit, } loader_func = getattr(_loader, "func", _loader) if inspect.iscoroutinefunction(loader_func): diff --git a/pytest_postgresql/plugin.py b/pytest_postgresql/plugin.py index 2057ce29..8612a79e 100644 --- a/pytest_postgresql/plugin.py +++ b/pytest_postgresql/plugin.py @@ -53,6 +53,11 @@ "when database was not cleared due to errors in previous test runs. " "Use cautiously and not on CI." ) +_help_load_autocommit = ( + "Run the SQL loader connection with autocommit enabled. " + "Required for statements that cannot run inside a transaction block, " + "e.g. CREATE DATABASE in a loaded .sql file." +) # psycopg async cannot use Windows' default ProactorEventLoop (the default since # Python 3.8). libpq socket I/O relies on selector APIs (add_reader / fileno) @@ -163,6 +168,7 @@ def pytest_addoption(parser: Parser) -> None: parser.addini(name="postgresql_load", type="pathlist", help=_help_load) parser.addini(name="postgresql_postgres_options", help=_help_postgres_options, default="") + parser.addini(name="postgresql_load_autocommit", type="bool", default=False, help=_help_load_autocommit) parser.addoption( "--postgresql-exec", @@ -225,6 +231,13 @@ def pytest_addoption(parser: Parser) -> None: help=_help_drop_test_database, ) + parser.addoption( + "--postgresql-load-autocommit", + action="store_true", + dest="postgresql_load_autocommit", + help=_help_load_autocommit, + ) + postgresql_proc = factories.postgresql_proc() postgresql_noproc = factories.postgresql_noproc() diff --git a/tests/examples/test_assert_load_autocommit_is_true.py b/tests/examples/test_assert_load_autocommit_is_true.py new file mode 100644 index 00000000..a9099088 --- /dev/null +++ b/tests/examples/test_assert_load_autocommit_is_true.py @@ -0,0 +1,14 @@ +"""Asserts that load_autocommit is True. + +That shows that it is not the default (False), and that we parsed it as a bool. +""" + +from pytest import FixtureRequest + +from pytest_postgresql.config import get_config + + +def test_assert_load_autocommit_is_true(request: FixtureRequest) -> None: + """Asserts that load_autocommit is True.""" + config = get_config(request) + assert config.load_autocommit is True diff --git a/tests/test_janitor.py b/tests/test_janitor.py index a38e8b8a..3e594534 100644 --- a/tests/test_janitor.py +++ b/tests/test_janitor.py @@ -134,6 +134,7 @@ def test_janitor_populate(connect_mock: MagicMock, load_database: str) -> None: "user": "user", "dbname": "database_name", "password": TEST_PASSWORD, + "autocommit": False, } janitor = DatabaseJanitor(version=10, **call_kwargs) # type: ignore[arg-type] janitor.load(load_database) @@ -156,6 +157,7 @@ async def test_janitor_populate_async(connect_mock: MagicMock, load_database: st "user": "user", "dbname": "database_name", "password": TEST_PASSWORD, + "autocommit": False, } janitor = AsyncDatabaseJanitor(version=10, **call_kwargs) # type: ignore[arg-type] await janitor.load(load_database) @@ -163,6 +165,39 @@ async def test_janitor_populate_async(connect_mock: MagicMock, load_database: st assert connect_mock.call_args.kwargs == call_kwargs +@patch("pytest_postgresql.janitor.psycopg.connect") +def test_janitor_load_forwards_autocommit(connect_mock: MagicMock) -> None: + """DatabaseJanitor.load forwards the autocommit flag to the loader connection.""" + janitor = DatabaseJanitor( + version=10, + host="host", + port="1234", + user="user", + dbname="database_name", + password=TEST_PASSWORD, + autocommit=True, + ) + janitor.load("tests.loader.load_database") + assert connect_mock.call_args.kwargs["autocommit"] is True + + +@pytest.mark.asyncio +@patch("tests.loader.psycopg.connect") +async def test_janitor_load_forwards_autocommit_async(connect_mock: MagicMock) -> None: + """AsyncDatabaseJanitor.load forwards the autocommit flag to the loader connection.""" + janitor = AsyncDatabaseJanitor( + version=10, + host="host", + port="1234", + user="user", + dbname="database_name", + password=TEST_PASSWORD, + autocommit=True, + ) + await janitor.load("tests.loader.load_database") + assert connect_mock.call_args.kwargs["autocommit"] is True + + @pytest.mark.asyncio async def test_janitor_populate_async_awaitable_loader() -> None: """AsyncDatabaseJanitor.load awaits async loader callables.""" @@ -172,6 +207,7 @@ async def test_janitor_populate_async_awaitable_loader() -> None: "user": "user", "dbname": "database_name", "password": TEST_PASSWORD, + "autocommit": False, } loader_mock = AsyncMock() @@ -192,6 +228,7 @@ async def test_janitor_populate_async_sync_loader_returns_awaitable() -> None: "user": "user", "dbname": "database_name", "password": TEST_PASSWORD, + "autocommit": False, } loader_mock = AsyncMock() diff --git a/tests/test_postgres_options_plugin.py b/tests/test_postgres_options_plugin.py index d7eec7cd..ee704e80 100644 --- a/tests/test_postgres_options_plugin.py +++ b/tests/test_postgres_options_plugin.py @@ -57,6 +57,21 @@ def test_postgres_loader_in_ini(pointed_pytester: Pytester) -> None: ret.assert_outcomes(passed=1) +def test_postgres_load_autocommit_in_cli(pointed_pytester: Pytester) -> None: + """Check that the --postgresql-load-autocommit command line flag is honored.""" + pointed_pytester.copy_example("test_assert_load_autocommit_is_true.py") + ret = pointed_pytester.runpytest("--postgresql-load-autocommit", "test_assert_load_autocommit_is_true.py") + ret.assert_outcomes(passed=1) + + +def test_postgres_load_autocommit_in_ini(pointed_pytester: Pytester) -> None: + """Check that pytest.ini postgresql_load_autocommit is honored and parsed as a bool.""" + pointed_pytester.copy_example("test_assert_load_autocommit_is_true.py") + pointed_pytester.makefile(".ini", pytest="[pytest]\npostgresql_load_autocommit = True\n") + ret = pointed_pytester.runpytest("test_assert_load_autocommit_is_true.py") + ret.assert_outcomes(passed=1) + + def test_postgres_port_search_count_in_cli_is_int(pointed_pytester: Pytester) -> None: """Check that the --postgresql-port-search-count command line argument is parsed as an int.""" pointed_pytester.copy_example("test_assert_port_search_count_is_ten.py") From e9bd138dff596eb7c006ed029060f3e2ab194d39 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Grzegorz=20=C5=9Aliwi=C5=84ski?= Date: Mon, 27 Jul 2026 21:15:08 +0200 Subject: [PATCH 2/4] Preserve factory False, update readme, break newsfragment --- README.rst | 2 +- newsfragments/902.break.rst | 2 ++ pytest_postgresql/factories/noprocess.py | 6 +++++- pytest_postgresql/factories/process.py | 7 +++++-- 4 files changed, 13 insertions(+), 4 deletions(-) create mode 100644 newsfragments/902.break.rst diff --git a/README.rst b/README.rst index 8122a964..145a8748 100644 --- a/README.rst +++ b/README.rst @@ -387,7 +387,7 @@ You can define settings via fixture factory arguments, command line options, or - load_autocommit - --postgresql-load-autocommit - postgresql_load_autocommit - - - + - yes - False * - PostgreSQL connection options - options diff --git a/newsfragments/902.break.rst b/newsfragments/902.break.rst new file mode 100644 index 00000000..64f4345d --- /dev/null +++ b/newsfragments/902.break.rst @@ -0,0 +1,2 @@ +`postgresql_noproc` has an additional parameter `load_autocommit` which is grouped together with `load` making +positional arguments filled till `depends_on` incompatible. diff --git a/pytest_postgresql/factories/noprocess.py b/pytest_postgresql/factories/noprocess.py index da5fbd40..eb0d046a 100644 --- a/pytest_postgresql/factories/noprocess.py +++ b/pytest_postgresql/factories/noprocess.py @@ -100,6 +100,10 @@ def postgresql_noproc_fixture(request: FixtureRequest) -> Iterator[NoopExecutor] else: noop_exec_dbname = pg_dbname + janitor_load_autocommit = config.load_autocommit + if load_autocommit is not None: + janitor_load_autocommit = load_autocommit + noop_exec = NoopExecutor( host=pg_host, port=pg_port, @@ -117,7 +121,7 @@ def postgresql_noproc_fixture(request: FixtureRequest) -> Iterator[NoopExecutor] as_template=True, version=noop_exec.version, password=noop_exec.password, - autocommit=load_autocommit or config.load_autocommit, + autocommit=janitor_load_autocommit, ) if drop_test_database: janitor.drop() diff --git a/pytest_postgresql/factories/process.py b/pytest_postgresql/factories/process.py index c0376429..a007f5cb 100644 --- a/pytest_postgresql/factories/process.py +++ b/pytest_postgresql/factories/process.py @@ -94,7 +94,7 @@ def postgresql_proc( unixsocketdir: str | None = None, postgres_options: str | None = None, load: list[Callable | str | Path] | None = None, - load_autocommit: bool = False, + load_autocommit: bool | None = None, ) -> Callable[[FixtureRequest, TempPathFactory], PostgreSQLExecutor]: """Postgresql process factory. @@ -216,6 +216,9 @@ def _cleanup_executor_resources() -> None: ) postgresql_executor.start() postgresql_executor.wait_for_postgres() + janitor_load_autocommit = config.load_autocommit + if load_autocommit is not None: + janitor_load_autocommit = load_autocommit janitor = DatabaseJanitor( user=postgresql_executor.user, host=postgresql_executor.host, @@ -224,7 +227,7 @@ def _cleanup_executor_resources() -> None: as_template=True, version=postgresql_executor.version, password=postgresql_executor.password, - autocommit=load_autocommit or config.load_autocommit, + autocommit=janitor_load_autocommit, ) if config.drop_test_database: janitor.drop() From d7ce8112bf2822e53c92c72a219289ff8cf905a3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Grzegorz=20=C5=9Aliwi=C5=84ski?= Date: Mon, 27 Jul 2026 21:22:06 +0200 Subject: [PATCH 3/4] Another break and test load functions fix --- newsfragments/902.break.1.rst | 1 + tests/test_chaining.py | 9 ++++++--- 2 files changed, 7 insertions(+), 3 deletions(-) create mode 100644 newsfragments/902.break.1.rst diff --git a/newsfragments/902.break.1.rst b/newsfragments/902.break.1.rst new file mode 100644 index 00000000..30efbee9 --- /dev/null +++ b/newsfragments/902.break.1.rst @@ -0,0 +1 @@ +Load functions passed to proc/noproc fixtures have to accept new `autocommit` argument. diff --git a/tests/test_chaining.py b/tests/test_chaining.py index ba700cae..d01377ed 100644 --- a/tests/test_chaining.py +++ b/tests/test_chaining.py @@ -7,26 +7,29 @@ from pytest_postgresql.executor_noop import NoopExecutor -def load_schema(host: str, port: int, user: str, dbname: str, password: str | None) -> None: +def load_schema(host: str, port: int, user: str, dbname: str, password: str | None, autocommit: bool) -> None: """Load schema into the database.""" with psycopg.connect(host=host, port=port, user=user, dbname=dbname, password=password) as conn: + conn.autocommit = autocommit with conn.cursor() as cur: cur.execute("CREATE TABLE schema_table (id serial PRIMARY KEY, name varchar);") conn.commit() -def load_data(host: str, port: int, user: str, dbname: str, password: str | None) -> None: +def load_data(host: str, port: int, user: str, dbname: str, password: str | None, autocommit: bool) -> None: """Load the first layer of data into the database.""" with psycopg.connect(host=host, port=port, user=user, dbname=dbname, password=password) as conn: + conn.autocommit = autocommit with conn.cursor() as cur: cur.execute("INSERT INTO schema_table (name) VALUES ('data_layer');") cur.execute("CREATE TABLE data_table (id serial PRIMARY KEY, val varchar);") conn.commit() -def load_more_data(host: str, port: int, user: str, dbname: str, password: str | None) -> None: +def load_more_data(host: str, port: int, user: str, dbname: str, password: str | None, autocommit: bool) -> None: """Load the second layer of data into the database.""" with psycopg.connect(host=host, port=port, user=user, dbname=dbname, password=password) as conn: + conn.autocommit = autocommit with conn.cursor() as cur: cur.execute("INSERT INTO schema_table (name) VALUES ('more_data_layer');") cur.execute("CREATE TABLE more_data_table (id serial PRIMARY KEY, extra varchar);") From 3c475da0a2db18dc9ab6eaa09dbb9003d504b01c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Grzegorz=20=C5=9Aliwi=C5=84ski?= Date: Mon, 27 Jul 2026 21:22:35 +0200 Subject: [PATCH 4/4] fix: noproc load_autocommit --- pytest_postgresql/factories/noprocess.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pytest_postgresql/factories/noprocess.py b/pytest_postgresql/factories/noprocess.py index eb0d046a..7c1eef48 100644 --- a/pytest_postgresql/factories/noprocess.py +++ b/pytest_postgresql/factories/noprocess.py @@ -45,7 +45,7 @@ def postgresql_noproc( dbname: str | None = None, options: str = "", load: list[Callable | str | Path] | None = None, - load_autocommit: bool = False, + load_autocommit: bool | None = None, depends_on: str | None = None, ) -> Callable[[FixtureRequest], Iterator[NoopExecutor]]: """Postgresql noprocess factory.