Refactor many slow functional tests to be faster

.github/workflows/scripts/func_test_script.sh

@@ -0,0 +1,24 @@
+#!/usr/bin/env bash
+# Functional test runner with task-queue sampling right after parallel suites.
+# Sourced from script.sh via FUNC_TEST_SCRIPT.
+
+_nightly_args=()
+if [[ "${GITHUB_WORKFLOW:-}" =~ "Nightly" ]]; then
+  _nightly_args=(--nightly)
+fi
+
+_pytest_common=(-v --timeout=300 -r sx --color=yes --suppress-no-test-exit-code --durations=20)
+
+# Parallel suites first (xdist -n 8), then sample worker-queue waits before serial.
+cmd_user_prefix bash -c "pytest ${_pytest_common[*]} --pyargs pulpcore.tests.functional -m parallel -n 8 ${_nightly_args[*]}"
+cmd_user_prefix bash -c "pytest ${_pytest_common[*]} --pyargs pulp_file.tests.functional -m parallel -n 8 ${_nightly_args[*]}"
+cmd_user_prefix bash -c "pytest ${_pytest_common[*]} --pyargs pulp_certguard.tests.functional -m parallel -n 8 ${_nightly_args[*]}"
+
+echo "::group::Task queue wait stats (after parallel suites)"
+# Short window: capture tasks from the parallel phase before purge/serial dilute the signal.
+cmd_user_prefix pulpcore-manager task-queue-stats --hours 1 --top 20 || true
+echo "::endgroup::"
+
+cmd_user_prefix bash -c "pytest ${_pytest_common[*]} --pyargs pulpcore.tests.functional -m 'not parallel' ${_nightly_args[*]}"
+cmd_user_prefix bash -c "pytest ${_pytest_common[*]} --pyargs pulp_file.tests.functional -m 'not parallel' ${_nightly_args[*]}"
+cmd_user_prefix bash -c "pytest ${_pytest_common[*]} --pyargs pulp_certguard.tests.functional -m 'not parallel' ${_nightly_args[*]}"

CHANGES/+task-queue-stats.misc

@@ -0,0 +1 @@
+Add `pulpcore-manager task-queue-stats` to summarize worker-queue waits from `unblocked_at`.

CLAUDE.md

@@ -38,6 +38,8 @@ pulpcore & pulp-file functional tests require both client bindings to be install
 
 **Always** use the `oci-env` to run the functional and unit tests.
 
+Do not UPDATE `Task.pulp_created`. A Postgres trigger (`on_update_timestamp_task`) raises `Updating pulp_created is not allowed.` Create rows in the desired order instead.
+
 ## Modifying template_config.yml
 
 Use the `plugin-template` tool after any changes made to `template_config.yml`.
@@ -57,6 +59,6 @@ When patchback fails to cherry-pick a PR into an older branch, you need to manua
 
 ## Contributing
 
-All docs, code comments, and changelogs are in markdown format. Keep comments and changelogs short and concise. Try to keep changelogs to just one line. 
+All docs, code comments, and changelogs are in markdown format. Keep comments and changelogs short and concise. Try to keep changelogs to just one line. Use single backticks for inline code (`like_this`), not RST double backticks (``like_this``).
 
 When preparing to commit and create a PR you **must** follow our [PR checklist](https://pulpproject.org/pulpcore/docs/dev/guides/pull-request-walkthrough/) Important to note is the AI attribution requirement in our commit messages. Also, note that our changelog entries are markdown.

pulp_file/tests/functional/api/test_auto_publish.py

@@ -1,5 +1,7 @@
 """Tests that sync file plugin repositories."""
 
+from uuid import uuid4
+
 import pytest
 
 from pulpcore.client.pulp_file import (
@@ -24,8 +26,10 @@ def test_auto_publish_and_distribution(
     file_random_content_unit,
     monitor_task,
     has_pulp_plugin,
+    random_artifact_factory,
 ):
     """Tests auto-publish and auto-distribution"""
+    # Remote is only needed to assert mirror=True is rejected with autopublish.
     remote = file_remote_ssl_factory(manifest_path=basic_manifest_path, policy="on_demand")
     repo = file_bindings.RepositoriesFileApi.read(file_repo_with_auto_publish.pulp_href)
     distribution = gen_object_with_cleanup(
@@ -44,12 +48,18 @@ def test_auto_publish_and_distribution(
     )
     assert distribution.publication is None
 
-    # Check what content and artifacts are in the fixture repository
-    expected_files = get_files_in_manifest(remote.url)
-
-    # Sync from the remote
-    body = FileRepositorySyncURL(remote=remote.pulp_href)
-    monitor_task(file_bindings.RepositoriesFileApi.sync(repo.pulp_href, body).task)
+    # One content unit is enough for version 1; attaching it triggers autopublish.
+    artifact = random_artifact_factory()
+    relative_path = f"{uuid4()}.iso"
+    created = monitor_task(
+        file_bindings.ContentFilesApi.create(
+            artifact=artifact.pulp_href,
+            relative_path=relative_path,
+            repository=repo.pulp_href,
+        ).task
+    ).created_resources
+    content = file_bindings.ContentFilesApi.read(created[1] if len(created) > 1 else created[0])
+    expected_files = {(relative_path, content.sha256, str(artifact.size))}
     repo = file_bindings.RepositoriesFileApi.read(repo.pulp_href)
 
     # Assert that a new repository version was created and a publication was created

pulp_file/tests/functional/api/test_filesystem_export.py

@@ -174,47 +174,64 @@ def test_fsexport_by_version(
     }
 
 
+def _filesystem_domain(pulpcore_bindings, gen_object_with_cleanup):
+    body = {
+        "name": str(uuid.uuid4()),
+        "storage_class": "pulpcore.app.models.storage.FileSystem",
+        "storage_settings": {"MEDIA_ROOT": "/var/lib/pulp/media/"},
+    }
+    return gen_object_with_cleanup(pulpcore_bindings.DomainsApi, body)
+
+
 @pytest.mark.skipif(not settings.DOMAIN_ENABLED, reason="Domains not enabled.")
 @pytest.mark.parallel
 def test_fsexport_cross_domain(
     fs_exporter_factory,
     fs_export_factory,
     gen_object_with_cleanup,
     pulpcore_bindings,
-    pub_and_repo,
+    file_bindings,
+    file_repository_factory,
+    file_publication_factory,
+    tmp_path,
+    monitor_task,
 ):
+    # Publication and versions live in source_domain; exporter lives in other_domain.
+    source_domain = _filesystem_domain(pulpcore_bindings, gen_object_with_cleanup)
+    other_domain = _filesystem_domain(pulpcore_bindings, gen_object_with_cleanup)
+
+    src = tmp_path / "file.dat"
+    src.write_text("x")
+    content_href = file_bindings.ContentFilesApi.upload(
+        relative_path="0.dat", file=str(src), pulp_domain=source_domain.name
+    ).pulp_href
+    repository = file_repository_factory(pulp_domain=source_domain.name)
+    monitor_task(
+        file_bindings.RepositoriesFileApi.modify(
+            repository.pulp_href, {"add_content_units": [content_href]}
+        ).task
+    )
+    repository = file_bindings.RepositoriesFileApi.read(repository.pulp_href)
+    publication = file_publication_factory(
+        repository=repository.pulp_href, pulp_domain=source_domain.name
+    )
+    latest = repository.latest_version_href
+    zeroth = latest.rsplit("/", 2)[0] + "/0/"
+    exporter = fs_exporter_factory(pulp_domain=other_domain.name)
 
-    entities = [{}, {}]
-    for e in entities:
-        body = {
-            "name": str(uuid.uuid4()),
-            "storage_class": "pulpcore.app.models.storage.FileSystem",
-            "storage_settings": {"MEDIA_ROOT": "/var/lib/pulp/media/"},
-        }
-        e["domain"] = gen_object_with_cleanup(pulpcore_bindings.DomainsApi, body)
-        e["publication"], e["repository"] = pub_and_repo(pulp_domain=e["domain"].name)
-        e["exporter"] = fs_exporter_factory(pulp_domain=e["domain"].name)
-        body = {"publication": e["publication"].pulp_href}
-        e["export"] = fs_export_factory(e["exporter"], body=body)
-
-    latest = entities[0]["repository"].latest_version_href
-    zeroth = latest.replace("/2/", "/0/")
-
-    with pytest.raises(BadRequestException) as e:
-        body = {"publication": entities[0]["publication"].pulp_href}
-        fs_export_factory(entities[1]["exporter"], body=body)
+    with pytest.raises(BadRequestException):
+        fs_export_factory(exporter, body={"publication": publication.pulp_href})
 
-    with pytest.raises(BadRequestException) as e:
-        body = {"repository_version": latest}
-        fs_export_factory(entities[1]["exporter"], body=body)
+    with pytest.raises(BadRequestException):
+        fs_export_factory(exporter, body={"repository_version": latest})
 
-    with pytest.raises(BadRequestException) as e:
-        body = {"repository_version": latest, "start_repository_version": zeroth}
-        fs_export_factory(entities[1]["exporter"], body=body)
+    with pytest.raises(BadRequestException):
+        fs_export_factory(
+            exporter, body={"repository_version": latest, "start_repository_version": zeroth}
+        )
 
-    with pytest.raises(BadRequestException) as e:
-        body = {
-            "publication": entities[0]["publication"].pulp_href,
-            "start_repository_version": zeroth,
-        }
-        fs_export_factory(entities[1]["exporter"], body=body)
+    with pytest.raises(BadRequestException):
+        fs_export_factory(
+            exporter,
+            body={"publication": publication.pulp_href, "start_repository_version": zeroth},
+        )

pulp_file/tests/functional/api/test_mime_types.py

@@ -13,30 +13,36 @@ def test_content_types(
     file_bindings,
     distribution_base_url,
     file_repo_with_auto_publish,
-    file_content_unit_with_name_factory,
     gen_object_with_cleanup,
     monitor_task,
+    tmp_path,
 ):
     """Test if content-app correctly returns mime-types based on filenames."""
+    relative_paths = {
+        "tar.gz": f"{uuid.uuid4()}.tar.gz",
+        "xml.gz": f"{uuid.uuid4()}.xml.gz",
+        "xml.bz2": f"{uuid.uuid4()}.xml.bz2",
+        "xml.zstd": f"{uuid.uuid4()}.xml.zstd",
+        "xml.xz": f"{uuid.uuid4()}.xml.xz",
+        "json.zstd": f"{uuid.uuid4()}.json.zstd",
+        "json": f"{uuid.uuid4()}.json",
+        "txt": f"{uuid.uuid4()}.txt",
+        "xml": f"{uuid.uuid4()}.xml",
+        "jpg": f"{uuid.uuid4()}.jpg",
+        "JPG": f"{uuid.uuid4()}.JPG",
+        "halabala": f"{uuid.uuid4()}.halabala",
+        "noextension1": f"{uuid.uuid4()}.asd/.asd/a",
+        "noextension2": f"{uuid.uuid4()}.....f",
+    }
+
+    blob = tmp_path / "blob"
+    blob.write_bytes(b"mime-type-test")
     files = {
-        "tar.gz": file_content_unit_with_name_factory(f"{str(uuid.uuid4())}.tar.gz"),
-        "xml.gz": file_content_unit_with_name_factory(f"{str(uuid.uuid4())}.xml.gz"),
-        "xml.bz2": file_content_unit_with_name_factory(f"{str(uuid.uuid4())}.xml.bz2"),
-        "xml.zstd": file_content_unit_with_name_factory(f"{str(uuid.uuid4())}.xml.zstd"),
-        "xml.xz": file_content_unit_with_name_factory(f"{str(uuid.uuid4())}.xml.xz"),
-        "json.zstd": file_content_unit_with_name_factory(f"{str(uuid.uuid4())}.json.zstd"),
-        "json": file_content_unit_with_name_factory(f"{str(uuid.uuid4())}.json"),
-        "txt": file_content_unit_with_name_factory(f"{str(uuid.uuid4())}.txt"),
-        "xml": file_content_unit_with_name_factory(f"{str(uuid.uuid4())}.xml"),
-        "jpg": file_content_unit_with_name_factory(f"{str(uuid.uuid4())}.jpg"),
-        "JPG": file_content_unit_with_name_factory(f"{str(uuid.uuid4())}.JPG"),
-        "halabala": file_content_unit_with_name_factory(f"{str(uuid.uuid4())}.halabala"),
-        "noextension1": file_content_unit_with_name_factory(f"{str(uuid.uuid4())}.asd/.asd/a"),
-        "noextension2": file_content_unit_with_name_factory(f"{str(uuid.uuid4())}.....f"),
+        extension: file_bindings.ContentFilesApi.upload(file=str(blob), relative_path=relative_path)
+        for extension, relative_path in relative_paths.items()
     }
 
-    units_to_add = list(map(lambda f: f.pulp_href, files.values()))
-    data = RepositoryAddRemoveContent(add_content_units=units_to_add)
+    data = RepositoryAddRemoveContent(add_content_units=[f.pulp_href for f in files.values()])
     monitor_task(
         file_bindings.RepositoriesFileApi.modify(file_repo_with_auto_publish.pulp_href, data).task
     )
@@ -49,18 +55,20 @@ def test_content_types(
     distribution = gen_object_with_cleanup(file_bindings.DistributionsFileApi, data)
     distribution_base_url = distribution_base_url(distribution.base_url)
 
-    received_mimetypes = {}
-    for extension, content_unit in files.items():
+    async def fetch_mimetypes():
+        async with aiohttp.ClientSession() as session:
 
-        async def get_content_type():
-            async with aiohttp.ClientSession() as session:
+            async def get_content_type(extension, content_unit):
                 url = urljoin(distribution_base_url, content_unit.relative_path)
                 async with session.get(url) as response:
-                    return response.headers.get("Content-Type")
+                    return extension, response.headers.get("Content-Type")
 
-        content_type = asyncio.run(get_content_type())
-        received_mimetypes[extension] = content_type
+            pairs = await asyncio.gather(
+                *(get_content_type(ext, unit) for ext, unit in files.items())
+            )
+        return dict(pairs)
 
+    received_mimetypes = asyncio.run(fetch_mimetypes())
     expected_mimetypes = {
         "tar.gz": "application/gzip",
         "xml.gz": "application/gzip",