diff --git a/.github/workflows/scripts/before_install.sh b/.github/workflows/scripts/before_install.sh index 95a1ba503cb..27cd6ec193d 100755 --- a/.github/workflows/scripts/before_install.sh +++ b/.github/workflows/scripts/before_install.sh @@ -55,7 +55,7 @@ pulp_scheme: "https" image: name: "pulp" tag: "ci_build" - ci_base: "ghcr.io/pulp/pulp-ci-centos9:latest" + ci_base: "ghcr.io/pulp/pulp-ci-centos10:latest" source: "${COMPONENT_SOURCE}" ci_requirements: $(test -f ci_requirements.txt && echo -n true || echo -n false) upperbounds: $(test "${TEST}" = "pulp" && echo -n true || echo -n false) diff --git a/CHANGES/+fix-add-signing-service.bugfix b/CHANGES/+fix-add-signing-service.bugfix new file mode 100644 index 00000000000..5e00ecf8b96 --- /dev/null +++ b/CHANGES/+fix-add-signing-service.bugfix @@ -0,0 +1 @@ +Fixed `add-signing-service` management command failing with "There are N keys matching the key id" for PGP keys that have subkeys. diff --git a/CHANGES/+gpg_verify.bugfix b/CHANGES/+gpg_verify.bugfix new file mode 100644 index 00000000000..06433d4faeb --- /dev/null +++ b/CHANGES/+gpg_verify.bugfix @@ -0,0 +1 @@ +Fixed an issue where gpg_verify() was rejecting some valid PGP signatures after changes made in pulpcore 3.108. diff --git a/CHANGES/+header-too-large.bugfix b/CHANGES/+header-too-large.bugfix new file mode 100644 index 00000000000..a5106eec27e --- /dev/null +++ b/CHANGES/+header-too-large.bugfix @@ -0,0 +1 @@ +Increased the content app's maximum HTTP header field size from 8190 to 16384 bytes to support PQC (post-quantum) X.509 certificates forwarded via the `X-CLIENT-CERT` header. diff --git a/CHANGES/7479.feature b/CHANGES/7479.feature new file mode 100644 index 00000000000..6dab6beeed6 --- /dev/null +++ b/CHANGES/7479.feature @@ -0,0 +1 @@ +Added `--backend` option to the `add-signing-service` management command, enabling Sequoia (`sq`) as an alternative to GPG for key management. Use `--backend sq` to register signing services using Sequoia's key store. diff --git a/MANIFEST.in b/MANIFEST.in index 862f03cc35e..fdda5e340fe 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -17,4 +17,5 @@ include test_requirements.txt exclude releasing.md exclude AGENTS.md exclude CLAUDE.md +exclude Makefile recursive-exclude pulpcore/tasking/task_trigger_demonstration * diff --git a/Makefile b/Makefile new file mode 100644 index 00000000000..b4228e5ba22 --- /dev/null +++ b/Makefile @@ -0,0 +1,20 @@ +# WARNING: DO NOT EDIT! +# +# This file was generated by plugin_template, and is managed by it. Please use +# './plugin-template --ci pulpcore' to update this file. +# +# For more info visit https://github.com/pulp/plugin_template + +.PHONY: format +format: + ruff format + ruff check --select I --fix + +.PHONY: lint +lint: + yamllint -s -d '{extends: relaxed, rules: {line-length: disable}}' .github/workflows + bump-my-version bump --dry-run --allow-dirty release + ruff format --check --diff + ruff check + check-manifest + python .ci/scripts/check_requirements.py diff --git a/functest_requirements.txt b/functest_requirements.txt index c8bb7e387ce..1a919656eab 100644 --- a/functest_requirements.txt +++ b/functest_requirements.txt @@ -4,6 +4,7 @@ pytest-xdist python-gnupg proxy.py~=2.4.10 trustme~=1.2.1 +cryptography>=49.0 # pulp_file tests beautifulsoup4 diff --git a/pulp_certguard/tests/functional/api/test_x509_certguard.py b/pulp_certguard/tests/functional/api/test_x509_certguard.py index bff81b03f67..3fad8a75109 100644 --- a/pulp_certguard/tests/functional/api/test_x509_certguard.py +++ b/pulp_certguard/tests/functional/api/test_x509_certguard.py @@ -1,4 +1,5 @@ import uuid +from collections import namedtuple from urllib.parse import quote, urljoin import pytest @@ -12,6 +13,11 @@ X509_UNTRUSTED_CLIENT_CERT_FILE_PATH, ) +PQCGuardedDistribution = namedtuple( + "PQCGuardedDistribution", + ["distribution", "algorithm", "client_cert_pem", "untrusted_client_cert_pem"], +) + @pytest.fixture(scope="class") def x509_certguard_factory(x509_content_guards_api_client, gen_object_with_cleanup): @@ -42,6 +48,31 @@ def x509_guarded_distribution( return distribution +@pytest.fixture(scope="class") +def pqc_guarded_distribution( + pqc_certificate_authority, + x509_content_guards_api_client, + gen_object_with_cleanup, + file_distribution_factory, + repository_test_file, +): + pca = pqc_certificate_authority + content_guard = gen_object_with_cleanup( + x509_content_guards_api_client, + { + "name": str(uuid.uuid4()), + "ca_certificate": pca.ca_cert_pem, + }, + ) + distribution = file_distribution_factory( + repository=repository_test_file.pulp_href, + content_guard=content_guard.pulp_href, + ) + return PQCGuardedDistribution( + distribution, pca.algorithm, pca.client_cert_pem, pca.untrusted_client_cert_pem + ) + + @pytest.fixture( scope="module", params=[ @@ -92,3 +123,50 @@ def test_download( headers=cert_data and {"X-CLIENT-CERT": cert_data}, ) assert response.status_code == status_code + + +class TestPQCX509CertGuard: + """Test X.509 content guard with PQC (ML-DSA) certificates. + + Parameterized over ML-DSA-65 (~7.5KB, under the default 8190-byte header + limit) and ML-DSA-87 (~10KB, over the limit). + """ + + def test_download_with_valid_cert( + self, + pqc_guarded_distribution, + distribution_base_url, + ): + distribution = pqc_guarded_distribution.distribution + url = distribution_base_url(distribution.base_url) + cert_pem = quote(pqc_guarded_distribution.client_cert_pem) + response = requests.get( + urljoin(url, "test_file"), + headers={"X-CLIENT-CERT": cert_pem}, + allow_redirects=False, + ) + assert response.status_code in (200, 302) + + def test_download_with_untrusted_cert( + self, + pqc_guarded_distribution, + distribution_base_url, + ): + distribution = pqc_guarded_distribution.distribution + url = distribution_base_url(distribution.base_url) + cert_pem = quote(pqc_guarded_distribution.untrusted_client_cert_pem) + response = requests.get( + urljoin(url, "test_file"), + headers={"X-CLIENT-CERT": cert_pem}, + ) + assert response.status_code == 403 + + def test_download_with_no_cert( + self, + pqc_guarded_distribution, + distribution_base_url, + ): + distribution = pqc_guarded_distribution.distribution + url = distribution_base_url(distribution.base_url) + response = requests.get(urljoin(url, "test_file")) + assert response.status_code == 403 diff --git a/pulp_certguard/tests/functional/constants.py b/pulp_certguard/tests/functional/constants.py index f481b1adb87..f0d04b2658a 100644 --- a/pulp_certguard/tests/functional/constants.py +++ b/pulp_certguard/tests/functional/constants.py @@ -14,7 +14,6 @@ X509_CERTS_BASE_PATH, "un_urlencoded_cert.txt" ) - RHSM_CA_CERT_FILE_PATH = os.path.join(_CURRENT_DIR, "artifacts", "rhsm", "katello-default-ca.crt") RHSM_CLIENT_CERT_FROM_UNTRUSTED_CA = os.path.join( diff --git a/pulp_file/pytest_plugin.py b/pulp_file/pytest_plugin.py index 62c83df846b..baa9f850830 100644 --- a/pulp_file/pytest_plugin.py +++ b/pulp_file/pytest_plugin.py @@ -255,6 +255,64 @@ def _file_remote_client_cert_req_factory(*, manifest_path, policy, **kwargs): return _file_remote_client_cert_req_factory +@pytest.fixture(scope="class") +def file_fixture_server_pqc_ssl(pqc_ssl_ctx, file_fixtures_root, gen_fixture_server): + return gen_fixture_server(file_fixtures_root, pqc_ssl_ctx) + + +@pytest.fixture(scope="class") +def file_fixture_server_pqc_ssl_client_cert_req( + pqc_ssl_ctx_req_client_auth, file_fixtures_root, gen_fixture_server +): + return gen_fixture_server(file_fixtures_root, pqc_ssl_ctx_req_client_auth) + + +@pytest.fixture(scope="class") +def file_remote_pqc_ssl_factory( + file_fixture_server_pqc_ssl, + file_bindings, + pqc_certificate_authority, + gen_object_with_cleanup, +): + def _file_remote_pqc_ssl_factory(*, manifest_path, policy, **kwargs): + url = file_fixture_server_pqc_ssl.make_url(manifest_path) + kwargs.update( + { + "url": str(url), + "policy": policy, + "name": str(uuid.uuid4()), + "ca_cert": pqc_certificate_authority.ca_cert_pem, + } + ) + return gen_object_with_cleanup(file_bindings.RemotesFileApi, kwargs) + + return _file_remote_pqc_ssl_factory + + +@pytest.fixture(scope="class") +def file_remote_pqc_client_cert_req_factory( + file_fixture_server_pqc_ssl_client_cert_req, + file_bindings, + pqc_certificate_authority, + gen_object_with_cleanup, +): + def _file_remote_pqc_client_cert_req_factory(*, manifest_path, policy, **kwargs): + url = file_fixture_server_pqc_ssl_client_cert_req.make_url(manifest_path) + kwargs.update( + { + "url": str(url), + "policy": policy, + "name": str(uuid.uuid4()), + "ca_cert": pqc_certificate_authority.ca_cert_pem, + "client_cert": pqc_certificate_authority.client_cert_pem, + "client_key": pqc_certificate_authority.client_key_pem, + } + ) + return gen_object_with_cleanup(file_bindings.RemotesFileApi, kwargs) + + return _file_remote_pqc_client_cert_req_factory + + @pytest.fixture(scope="class") def file_repository_factory(file_bindings, gen_object_with_cleanup): """A factory to generate a File Repository with auto-deletion after the test run.""" diff --git a/pulp_file/tests/functional/api/test_remote_settings.py b/pulp_file/tests/functional/api/test_remote_settings.py index d4d37ff2525..17a93395fae 100644 --- a/pulp_file/tests/functional/api/test_remote_settings.py +++ b/pulp_file/tests/functional/api/test_remote_settings.py @@ -217,6 +217,52 @@ def test_http_sync_ssl_with_client_cert_req( ) +@pytest.mark.parallel +def test_http_sync_pqc_ssl_tls_validation_on( + file_bindings, + file_remote_pqc_ssl_factory, + file_repo, + basic_manifest_path, + monitor_task, +): + """ + Test file on_demand sync with https:// using PQC (ML-DSA-65) server certificate. + """ + remote_on_demand = file_remote_pqc_ssl_factory( + manifest_path=basic_manifest_path, policy="on_demand", tls_validation=True + ) + + _run_basic_sync_and_assert( + file_bindings, + remote_on_demand, + file_repo, + monitor_task, + ) + + +@pytest.mark.parallel +def test_http_sync_pqc_ssl_with_client_cert_req( + file_bindings, + file_remote_pqc_client_cert_req_factory, + file_repo, + basic_manifest_path, + monitor_task, +): + """ + Test file on_demand sync with https:// using PQC (ML-DSA-65) mutual TLS authentication. + """ + remote_on_demand = file_remote_pqc_client_cert_req_factory( + manifest_path=basic_manifest_path, policy="on_demand" + ) + + _run_basic_sync_and_assert( + file_bindings, + remote_on_demand, + file_repo, + monitor_task, + ) + + @pytest.mark.parallel def test_ondemand_to_immediate_sync( file_bindings, diff --git a/pulpcore/app/management/commands/add-signing-service.py b/pulpcore/app/management/commands/add-signing-service.py index 9cb4f1b32b5..44f74d9dc18 100644 --- a/pulpcore/app/management/commands/add-signing-service.py +++ b/pulpcore/app/management/commands/add-signing-service.py @@ -1,14 +1,20 @@ import os +import subprocess +import warnings from gettext import gettext as _ from pathlib import Path -import gnupg from django.apps import apps from django.core.management import BaseCommand, CommandError from django.db.utils import IntegrityError from pulpcore.app.models.content import SigningService as BaseSigningService +ENV_DEFAULTS = { + "gpg": "GNUPGHOME", + "sq": "SEQUOIA_HOME", +} + class Command(BaseCommand): """ @@ -28,7 +34,7 @@ def add_arguments(self, parser): ) parser.add_argument( "key", - help=_("Key id of the public key."), + help=_("Key id or fingerprint of the public key."), ) parser.add_argument( "--class", @@ -36,11 +42,27 @@ def add_arguments(self, parser): required=False, help=_("Signing service class prefixed by the app label separated by a colon."), ) + parser.add_argument( + "--backend", + choices=["gpg", "sq"], + default="gpg", + required=False, + help=_("Key management backend to use for extracting key metadata. (default: gpg)"), + ) + parser.add_argument( + "--home", + default=None, + required=False, + help=_( + "Home directory for the key management backend. " + "Defaults to $GNUPGHOME (gpg) or $SEQUOIA_HOME (sq)." + ), + ) parser.add_argument( "--gnupghome", - default=os.getenv("GNUPGHOME", ""), + default=None, required=False, - help=_("A default GnuPG home directory to use during the initialization."), + help=_("Deprecated: use --home instead."), ) parser.add_argument( "--keyring", @@ -68,13 +90,28 @@ def handle(self, *args, **options): ) ) - gpg = gnupg.GPG(gnupghome=options["gnupghome"], keyring=options["keyring"]) + backend = options["backend"] + + if options["home"] and options["gnupghome"]: + raise CommandError(_("--home and --gnupghome are mutually exclusive.")) + + if options["gnupghome"]: + warnings.warn( + "--gnupghome is deprecated; use --home instead.", + DeprecationWarning, + stacklevel=2, + ) + + home = options["home"] or options["gnupghome"] or os.getenv(ENV_DEFAULTS[backend], "") - key_list = gpg.list_keys(keys=[key_id]) - if not len(key_list) == 1: - raise CommandError(_("There are {} keys matching the key id.").format(len(key_list))) - fingerprint = key_list[0]["fingerprint"] - public_key = gpg.export_keys(key_id) + if backend == "sq": + fingerprint, public_key = self._extract_key_from_sq( + key_id, home, options.get("keyring") + ) + else: + fingerprint, public_key = self._extract_key_from_gpg( + key_id, home, options.get("keyring") + ) try: script_path = Path(script).resolve(strict=True) @@ -96,3 +133,70 @@ def handle(self, *args, **options): name=name, fingerprint=fingerprint ) ) + + def _extract_key_from_gpg(self, key_id, home, keyring): + gpg_cmd = ["gpg"] + if home: + gpg_cmd += ["--homedir", home] + if keyring: + gpg_cmd += ["--keyring", keyring] + + result = subprocess.run( + gpg_cmd + ["--with-colons", "--fingerprint", key_id], + capture_output=True, + text=True, + ) + if result.returncode != 0: + raise CommandError(result.stderr.strip()) + + lines = result.stdout.splitlines() + + # Count actual keys (pub:/sec: lines), not fingerprint lines. GPG emits + # a separate fpr: line for the primary key and each subkey, so a single + # key with subkeys produces multiple fpr: lines. + key_lines = [l for l in lines if l.startswith(("pub:", "sec:"))] # noqa: E741 + if len(key_lines) != 1: + raise CommandError(_("There are {} keys matching the key id.").format(len(key_lines))) + + # Use the primary key fingerprint (first fpr: line in GPG's output). + fingerprint = [l.split(":")[9] for l in lines if l.startswith("fpr:")][0] # noqa: E741 + + result = subprocess.run( + gpg_cmd + ["--armor", "--export", key_id], + capture_output=True, + text=True, + ) + if result.returncode != 0: + raise CommandError(result.stderr.strip()) + public_key = result.stdout + + return fingerprint, public_key + + def _extract_key_from_sq(self, key_id, home, keyring): + from pysequoia import Cert + + sq_cmd = ["sq"] + if home: + sq_cmd += ["--home", home] + if keyring: + sq_cmd += ["--keyring", keyring] + + result = subprocess.run( + sq_cmd + ["cert", "export", "--cert", key_id], + capture_output=True, + text=True, + ) + if result.returncode != 0: + raise CommandError(result.stderr.strip()) + public_key = result.stdout + + try: + cert = Cert.from_bytes(public_key.encode("utf-8")) + except Exception as e: + raise CommandError( + _("Failed to parse exported certificate for '{}': {}").format(key_id, e) + ) + + fingerprint = cert.fingerprint.upper() + + return fingerprint, public_key diff --git a/pulpcore/app/models/openpgp.py b/pulpcore/app/models/openpgp.py index b6d2b8eaf62..3c96e03e8f0 100644 --- a/pulpcore/app/models/openpgp.py +++ b/pulpcore/app/models/openpgp.py @@ -3,9 +3,9 @@ from aiohttp.web_response import Response from django.db import models from django.utils import timezone +from pysequoia import ArmorKind, armor from pulpcore.app.models import AutoAddObjPermsMixin, Content, Distribution, Repository -from pulpcore.app.openpgp import wrap_armor from pulpcore.app.util import get_domain_pk, gpg_verify @@ -48,6 +48,7 @@ def represent(self, repository_version=None): else: content_filter = {} data = self.packet() + # note: because these queries aren't ordered, the result may be nondetermininistic for signature in self.openpgp_signatures.filter(**content_filter): data += signature.packet() for user_id in self.user_ids.filter(**content_filter): @@ -62,7 +63,7 @@ def represent(self, repository_version=None): data += public_subkey.packet() for signature in public_subkey.openpgp_signatures.filter(**content_filter): data += signature.packet() - return wrap_armor(data) + return armor(data, ArmorKind.PublicKey).strip() # avoid trailing newline class Meta: default_related_name = "%(app_label)s_%(model_name)s" @@ -128,11 +129,11 @@ class OpenPGPSignature(_OpenPGPContent): sha256 = models.CharField(max_length=128) signature_type = models.PositiveSmallIntegerField() - created = models.DateTimeField() # 2 - expiration_time = models.DurationField(null=True) # 3 - key_expiration_time = models.DurationField(null=True) # 9 - issuer = models.CharField(max_length=16, null=True) # 16 - signers_user_id = models.CharField(null=True) # 28 + created = models.DateTimeField() + expiration_time = models.DurationField(null=True) + key_expiration_time = models.DurationField(null=True) + issuer = models.CharField(max_length=16, null=True) + signers_user_id = models.CharField(null=True) signed_content = models.ForeignKey( Content, related_name="openpgp_signatures", on_delete=models.PROTECT ) diff --git a/pulpcore/app/openpgp.py b/pulpcore/app/openpgp.py index f25639ce5b0..7335970bd99 100644 --- a/pulpcore/app/openpgp.py +++ b/pulpcore/app/openpgp.py @@ -1,529 +1,83 @@ import hashlib -from base64 import b64decode, b64encode -from django.utils import timezone - -# Source of information: -# * rfc4880 -# * rfc4880bis -# * gnupg sources -# * https://datatracker.ietf.org/doc/html/draft-shaw-openpgp-hkp-00 - - -PACKET_TYPES = { - 1: "Public-Key Encrypted Session Key Packet", - 2: "Signature Packet", - 3: "Symmetric-Key Encrypted Session Key Packet", - 4: "One-Pass Signature Packet", - 5: "Secret-Key Packet", - 6: "Public-Key Packet", - 7: "Secret-Subkey Packet", - 8: "Compressed Data Packet", - 9: "Symmetrically Encrypted Data Packet", - 10: "Marker Packet", - 11: "Literal Data Packet", - 12: "Trust Packet", - 13: "User ID Packet", - 14: "Public-Subkey Packet", - 17: "User Attribute Packet", - 18: "Sym. Encrypted and Integrity Protected Data Packet", - 19: "Modification Detection Code Packet", - 20: "OCB Encrypted Data Packet", -} - - -SIG_SUBPACKAGE_TYPES = { - 2: "Signature Creation Time", - 3: "Signature Expiration Time", - 4: "Exportable Certification", - 5: "Trust Signature", - 6: "Regular Expression", - 7: "Revocable", - 9: "Key Expiration Time", - 10: "Placeholder for backward compatibility", - 11: "Preferred Symmetric Algorithms", - 12: "Revocation Key", - 16: "Issuer", - 20: "Notation Data", - 21: "Preferred Hash Algorithms", - 22: "Preferred Compression Algorithms", - 23: "Key Server Preferences", - 24: "Preferred Key Server", - 25: "Primary User ID", - 26: "Policy URI", - 27: "Key Flags", - 28: "Signer's User ID", - 29: "Reason for Revocation", - 30: "Features", - 31: "Signature Target", - 32: "Embedded Signature", - 33: "Issuer Fingerprint", - 34: "Preferred Encryption Modes", - 35: "Intended Recipient Fingerprint", - 37: "Attested Certifications", - 38: "Key Block", - 40: "Literal Data Meta Hash", - 41: "Trust Alias", -} - - -PUBKEY_ALGORITHMS = { - 1: { - "name": "RSA", - "format": "mm", - }, - 2: { - "name": "RSA Encrypt-Only", - "format": "mm", - }, - 3: { - "name": "RSA Sign-Only", - "format": "mm", - }, - 16: { - "name": "Elgamal", - "format": "mmm", - }, - 17: { - "name": "DSA", - "format": "mmmm", - }, - 18: { - "name": "ECDH", - "format": "omk", - }, - 19: { - "name": "ECDSA", - "format": "om", - }, - 22: { - "name": "EdDSA", - "format": "om", - }, -} - - -HASH_ALGORITHMS = { - 1: "md5", - 2: "sha1", - 3: "ripemd160", - 8: "sha256", - 9: "sha384", - 10: "sha512", - 11: "sha224", - 12: "sha3-256", - 14: "sha3-512", -} - - -SYMMETRIC_ALGORITHMS = { - 0: "Plaintext", - 1: "IDEA", - 2: "TripleDES", - 3: "CAST5", - 4: "Blowfish", - 7: "AES128", - 8: "AES192", - 9: "AES256", - 10: "Twofish", - 11: "Camellia 128", - 12: "Camellia 192", - 13: "Camellia 256", -} - - -COMPRESSION_ALGORITHMS = { - 0: "Uncompressed", - 1: "ZIP", - 2: "ZLIB", - 3: "BZip2", -} - - -ENCRYPTION_MODES = { - 1: "EAX", - 2: "OCB", -} - - -def packet_iter(data): - begin = 0 - pos = 0 - while pos < len(data): - packet_tag = data[pos] - pos += 1 - if not packet_tag & 0x80: - raise ValueError("Invalid Packet Tag") - new_format = bool(packet_tag & 0x40) - - if new_format: - packet_type = packet_tag & 0x1F - - if data[pos] < 0xC0: - # 1-octet length - length = data[pos] - pos += 1 - elif data[pos] < 0xE0: - # 2-octet length - length = ((data[pos] - 0xC0) << 8) + data[pos + 1] + 0xC0 - pos += 2 - elif data[pos] == 0xFF: - # 5-octet length - length = int.from_bytes(data[pos + 1 : pos + 5], "big") - pos += 5 - else: - # Partial body length - raise NotImplementedError("Partial Packet Body is not implemented") - else: - packet_type = (packet_tag & 0x3C) >> 2 - length_type = packet_tag & 0x03 - if length_type == 3: - # Indeterminate packet length - length = len(data) - pos - else: - length_bytes = 1 << length_type - length = int.from_bytes(data[pos : pos + length_bytes], "big") - pos += length_bytes - - if packet_type == 0: - raise ValueError("Invalid Packet Type") - yield { - "type": packet_type, - "body": data[pos : pos + length], - "raw": data[begin : pos + length], - } - pos += length - begin = pos - if pos != len(data): - raise ValueError("Broken Stream") - - -def subpacket_iter(data): - begin = 0 - pos = 0 - while pos < len(data): - if data[pos] < 0xC0: - # 1-octet length - length = data[pos] - pos += 1 - elif data[pos] < 0xD0: - # 2-octet length - length = ((data[pos] - 0xC0) << 8) + data[pos + 1] + 0xC0 - pos += 2 - elif data[pos] == 0xFF: - # 5-octet length - length = int.from_bytes(data[pos + 1 : pos + 5], "big") - pos += 5 - else: - raise ValueError("Partial packet lengths are not allowed.") - yield { - "type": data[pos] & 0x7F, - "critical": bool(data[pos] & 0x80), - "body": data[pos + 1 : pos + length], - "raw": data[begin : pos + length], - } - pos += length - begin = pos - if pos != len(data): - raise ValueError("Broken Stream") - - -def extract_mpi(data): - bit_length = int.from_bytes(data[0 : 0 + 2], "big") - length = (bit_length + 7) // 8 - return {"bit_length": bit_length, "body": data[2 : 2 + length], "raw": data[0 : 2 + length]} - - -def extract_oid_kdf(data): - # These two types use the same method to decribe the length, which is all we want. - length = data[0] - return {"body": data[1 : 1 + length], "raw": data[0 : 1 + length]} - - -def analyze_sig_subpackets(data): - signature_attributes = {} - for packet in subpacket_iter(data): - packet_type = packet["type"] - body = packet["body"] - if packet_type == 2: - signature_attributes["created"] = timezone.datetime.fromtimestamp( - int.from_bytes(body, "big") - ).astimezone() - elif packet_type == 3: - signature_attributes["expiration_time"] = timezone.timedelta( - seconds=int.from_bytes(body, "big") - ) - elif packet_type == 9: - signature_attributes["key_expiration_time"] = timezone.timedelta( - seconds=int.from_bytes(body, "big") - ) - elif packet_type == 16: - signature_attributes["issuer"] = body.hex() - elif packet_type == 28: - signature_attributes["signers_user_id"] = body.decode() - return signature_attributes - - -def analyze_signature(data, pubkey, signed_packet_type, signed_packet): - # Type 2 - version = data[0] - if version == 3: - raise NotImplementedError("Version 3 signatures are not implemented.") - elif version in [4, 5]: - signature_type = data[1] - # pubkey_algorithm = PUBKEY_ALGORITHMS.get(data[2]) # Unused here. - hash_algorithm = HASH_ALGORITHMS.get(data[3]) - hashed_size = (data[4] << 8) + data[5] - hashed_data = data[6 : 6 + hashed_size] - unhashed_size = (data[6 + hashed_size] << 8) + data[7 + hashed_size] - unhashed_data = data[8 + hashed_size : 8 + hashed_size + unhashed_size] - canary = data[8 + hashed_size + unhashed_size : 10 + hashed_size + unhashed_size] - # signature = data[10 + hashed_size + unhashed_size :] # Unused here. - - if signature_type in [0x18, 0x19, 0x28]: - # 0x18 Subkey Binding Signature - # 0x19 Primary Key Binding Signature - # 0x28 Subkey Revocation Signature - if signed_packet_type != 14: - raise ValueError("Out of band subkey key signature.") - if version == 4: - hash_payload = b"\x99" + len(signed_packet).to_bytes(2, "big") + signed_packet - else: # version == 5 - hash_payload = b"\x9a" + len(signed_packet).to_bytes(4, "big") + signed_packet - elif signature_type in [0x10, 0x11, 0x12, 0x13, 0x16, 0x30]: - # 0x10 - 0x13 Certification of a user id or attribute - # 0x16 Attested Key Signature - # 0x30 Certification Revocation Signature - if signed_packet_type == 13: - hash_payload = b"\xb4" + len(signed_packet).to_bytes(4, "big") + signed_packet - elif signed_packet_type == 17: - hash_payload = b"\xd1" + len(signed_packet).to_bytes(4, "big") + signed_packet - else: - raise ValueError("Out of band user ID or attribute signature.") - elif signature_type in [0x1F, 0x20, 0x30]: - # 0x1F Direct Key Signature - # 0x20 Key Revocation Signature - # 0x30 Certification Revocation Signature - if signed_packet_type != 6: - raise ValueError("Out of band key signature.") - hash_payload = b"" - else: - # 0x50 Third-Party Confirmation Signature (does this even apply to keys?) - raise NotImplementedError(f"Unsupported signature type {signature_type:#x}.") - - # Validate the signature against the canary value - if hash_algorithm is None: - raise ValueError(f"Unknown hash algorithm {data[3]:#x} used for signature.") - h = hashlib.new(hash_algorithm) - if version == 4: - h.update(b"\x99" + len(pubkey).to_bytes(2, "big") + pubkey) - else: # version == 5 - h.update(b"\x9a" + len(pubkey).to_bytes(4, "big") + pubkey) - h.update(hash_payload) - if version == 4: - h.update( - data[: 6 + hashed_size] - + b"\x04\xff" - + ((6 + hashed_size) % (1 << 32)).to_bytes(4, "big") - ) - else: # version == 5 - h.update( - data[: 6 + hashed_size] - + b"\x05\xff" - + ((6 + hashed_size) % (1 << 64)).to_bytes(8, "big") - ) - if not h.digest().startswith(canary): - raise ValueError("Signature canary mismatch") - - # Hash the signature packet for db-uniqueness - sha256 = hashlib.sha256(data).hexdigest() - signature_attributes = { - "sha256": sha256, - "signature_type": signature_type, - "raw_data": data, - } - # Hashed Subpackets - signature_attributes.update(analyze_sig_subpackets(hashed_data)) - # Unhashed Subpackets - signature_attributes.update(analyze_sig_subpackets(unhashed_data)) - return signature_attributes - else: - raise ValueError(f"Invalid Packet version {version}") - - -def analyze_user_id(data): - # Type 13 - user_id = data.decode() - return {"raw_data": data, "user_id": user_id} - - -def analyze_user_attribute(data): - # Type 17 - # Treat as an opaque packet for now. - sha256 = hashlib.sha256(data).hexdigest() - return {"raw_data": data, "sha256": sha256} - - -def analyze_pubkey(data): - # Type 5, 6, 7 or 14 - # Type 5 and 7 are actually secret key packages. They begin with the corresponding public key - # package. Secret bits are ignored by us here. - version = data[0] - created = timezone.datetime.fromtimestamp(int.from_bytes(data[1:5], "big")).astimezone() - if version == 3: - n = extract_mpi(data[8:]) - e = extract_mpi(data[8 + len(n["raw"])]) - fingerprint = hashlib.md5(n["body"] + e["body"]).hexdigest() - # expiration = int.from_bytes(data[5:7], "big") # Unused here. Kept for documentation. - pubkey_algorithm = PUBKEY_ALGORITHMS.get(data[7]) - elif version in [4, 5]: - pubkey_algorithm = PUBKEY_ALGORITHMS.get(data[5]) - if version == 4: - key_data = data[6:] - else: - key_data_len = int.from_bytes(data[6:10], "big") - key_data = data[10 : 10 + key_data_len] - pub_key_body = data[: 10 + key_data_len] - fingerprint = hashlib.sha256( - b"\x9a" + len(pub_key_body).to_bytes(4, "big") + pub_key_body - ).hexdigest() - if pubkey_algorithm and "format" in pubkey_algorithm: - pos = 0 - for item_type in pubkey_algorithm["format"]: - if item_type == "m": - # Multi precision integer - mpi = extract_mpi(key_data[pos:]) - pos += len(mpi["raw"]) - elif item_type == "o": - # OID - oid = extract_oid_kdf(key_data[pos:]) - pos += len(oid["raw"]) - elif item_type == "k": - # KDF parameters - kdf = extract_oid_kdf(key_data[pos:]) - pos += len(kdf["raw"]) - else: - raise RuntimeError("Unknown key material format.") - if version == 4: - key_data_len = pos - pub_key_body = data[: 6 + key_data_len] - fingerprint = hashlib.sha1( - b"\x99" + len(pub_key_body).to_bytes(2, "big") + pub_key_body - ).hexdigest() - else: - if version == 4: - # We needed to analyse the public key algorithm to calculate the fingerprint of - # version 4 keys. Version 5 keys do not have this limitation, and we can get away - # with an unknown algorithm. - raise ValueError("Unknown public key algorithm.") - else: - raise ValueError(f"Invalid Packet version {version}") - return {"raw_data": data, "created": created, "fingerprint": fingerprint} - - -def gpg_crc24(data): - crc = 0xB704CE - for byte in data: - crc ^= byte << 16 - for i in range(8): - crc <<= 1 - if crc & 0x1000000: - crc ^= 0x1864CFB - return (crc & 0xFFFFFF).to_bytes(3, "big") - - -def unwrap_armor(data): - try: - lines = data.decode().strip().split("\n") - except UnicodeDecodeError: - # assume raw binary data - return data - line = lines.pop(0).strip() - if line.startswith("-----BEGIN ") and line.endswith("-----"): - message_type = line[11:-5] - else: - # Header not found assume raw binary data - return data - line = lines.pop(0).strip() - while line != "": - # Armor Headers - line = lines.pop(0).strip() - armor = "" - while line != "-----END " + message_type + "-----": - armor += line - line = lines.pop(0).strip() - if armor[-5] != "=": - raise ValueError("Broken Stream") - raw = b64decode(armor[:-5]) - checksum = b64decode(armor[-4:]) - if gpg_crc24(raw) != checksum: - raise ValueError("Checksum Mismatch") - return raw - - -def wrap_armor(raw, message_type="PGP PUBLIC KEY BLOCK"): - checksum = "=" + b64encode(gpg_crc24(raw)).decode() - data = b64encode(raw).decode() - lines = ["-----BEGIN " + message_type + "-----", ""] - while data: - line = data[:76] - data = data[76:] - lines.append(line) - lines.append(checksum) - lines.append("-----END " + message_type + "-----") - return "\n".join(lines) +from pysequoia.packet import PacketPile, Tag def read_public_key(data): - data = unwrap_armor(data) - packets = packet_iter(data) - - # The first packet must be the public key. - packet = next(packets) - if packet["type"] != 6: - raise ValueError("Not a public key.") - public_key = analyze_pubkey(packet["body"]) - public_key.update( - {"user_ids": [], "user_attributes": [], "public_subkeys": [], "signatures": []} - ) - signed_content = public_key - signed_packet_type = 6 - - for packet in packets: - packet_type = packet["type"] - body = packet["body"] - if packet_type == 2: - signed_content["signatures"].append( - analyze_signature( - body, - public_key["raw_data"], - signed_packet_type, - signed_content["raw_data"], - ) - ) + pile = PacketPile.from_bytes(data) + + public_key = None + signed_content = None + + for packet in pile: + tag = packet.tag + body = bytes(packet.body) + + if tag == Tag.PublicKey: + if public_key is not None: + raise ValueError("Multiple public keys found.") + public_key = { + "raw_data": body, + "fingerprint": packet.fingerprint, + "created": packet.key_created, + "user_ids": [], + "user_attributes": [], + "public_subkeys": [], + "signatures": [], + } + signed_content = public_key + + elif tag == Tag.PublicSubkey: + public_subkey = { + "raw_data": body, + "fingerprint": packet.fingerprint, + "created": packet.key_created, + "signatures": [], + } + signed_content = public_subkey + public_key["public_subkeys"].append(public_subkey) - elif packet_type == 13: - user_id = analyze_user_id(body) - user_id["signatures"] = [] + elif tag == Tag.UserID: + user_id = { + "raw_data": body, + "user_id": packet.user_id, + "signatures": [], + } signed_content = user_id - signed_packet_type = packet_type public_key["user_ids"].append(user_id) - elif packet_type == 14: - public_subkey = analyze_pubkey(body) - public_subkey["signatures"] = [] - signed_content = public_subkey - signed_packet_type = packet_type - public_key["public_subkeys"].append(public_subkey) - - elif packet_type == 17: - user_attribute = analyze_user_attribute(body) - user_attribute["signatures"] = [] + elif tag == Tag.UserAttribute: + user_attribute = { + "raw_data": body, + "sha256": hashlib.sha256(body).hexdigest(), + "signatures": [], + } signed_content = user_attribute - signed_packet_type = packet_type public_key["user_attributes"].append(user_attribute) + elif tag == Tag.Signature: + sig_attrs = { + "sha256": hashlib.sha256(body).hexdigest(), + "signature_type": body[1], + "raw_data": body, + "created": packet.signature_created, + } + # Note: Pulp's concept of "expiration time" is a duration starting at creation time + # Sequoia calls that "validity_period", while "expiration_time" is an instant + if packet.signature_validity_period is not None: + sig_attrs["expiration_time"] = packet.signature_validity_period + if packet.key_validity_period is not None: + sig_attrs["key_expiration_time"] = packet.key_validity_period + if packet.issuer_key_id is not None: + sig_attrs["issuer"] = packet.issuer_key_id + if packet.signers_user_id is not None: + sig_attrs["signers_user_id"] = packet.signers_user_id + signed_content["signatures"].append(sig_attrs) + else: - raise NotImplementedError("Invalid or unknown rfc4880 packet.") + raise NotImplementedError(f"Unexpected packet type: {tag!r}") + + if public_key is None: + raise ValueError("Not a public key.") return public_key diff --git a/pulpcore/app/util.py b/pulpcore/app/util.py index 8ee0a1a66db..11c6698fbc4 100644 --- a/pulpcore/app/util.py +++ b/pulpcore/app/util.py @@ -1,7 +1,6 @@ import hashlib import os import socket -import tempfile import zlib from contextlib import ExitStack from datetime import timedelta @@ -14,7 +13,6 @@ from urllib.parse import urlparse from uuid import UUID -import gnupg from django.apps import apps from django.conf import settings from django.db import connection @@ -383,9 +381,36 @@ def get_request_without_query_params(context): return request +class VerifyResult: + """ + Verification result mimicking the interface of gnupg.Verify for compatibility. + + Attributes: + valid (bool): Always True; invalid signatures raise InvalidSignatureError instead. + fingerprint (str): Fingerprint of the signing subkey (uppercase hex). + pubkey_fingerprint (str): Fingerprint of the signing certificate (uppercase hex). + key_id (str): Short (16-char) key ID derived from the fingerprint. + data (bytes or None): The verified plaintext content for inline signatures, None for + detached signatures. + """ + + def __init__(self, decrypted): + self.valid = True + self.data = bytes(decrypted.bytes) if decrypted.bytes is not None else None + vs = decrypted.valid_sigs[0] + self.fingerprint = vs.signing_key.upper() + self.pubkey_fingerprint = vs.certificate.upper() + self.key_id = vs.signing_key[-16:].upper() + + def __repr__(self): + return ( + f"" + ) + + def gpg_verify(public_keys, signature, detached_data=None): """ - Check whether the provided gnupg signature is valid for one of the provided public keys. + Check whether the provided signature is valid for one of the provided public keys. Args: public_keys (str): Ascii armored public key data @@ -394,25 +419,37 @@ def gpg_verify(public_keys, signature, detached_data=None): signature Returns: - gnupg.Verify: The result of the verification + VerifyResult: The verification result with `valid`, `fingerprint`, `pubkey_fingerprint`, + `key_id`, `username`, and `data` attributes, mimicking the gnupg.Verify interface. Raises: pulpcore.exceptions.validation.InvalidSignatureError: In case the signature is invalid. """ - with tempfile.TemporaryDirectory(dir=settings.WORKING_DIRECTORY) as temp_directory_name: - gpg = gnupg.GPG(gnupghome=temp_directory_name) - gpg.import_keys(public_keys) - - with ExitStack() as stack: - if isinstance(signature, str): - signature = stack.enter_context(open(signature, "rb")) - elif isinstance(signature, models.Artifact): - signature = stack.enter_context(signature.file) - - verified = gpg.verify_file(signature, detached_data) - if not verified.valid: - raise InvalidSignatureError(_("The signature is not valid."), verified=verified) - return verified + from pysequoia import Cert, Sig, verify + + certs = Cert.split_bytes(public_keys.encode("utf8")) + + def store(key_ids): + return certs + + with ExitStack() as stack: + if isinstance(signature, str): + sig_data = stack.enter_context(open(signature, "rb")).read() + elif isinstance(signature, models.Artifact): + sig_data = stack.enter_context(signature.file).read() + else: + sig_data = signature.read() + + try: + if detached_data is not None: + sig = Sig.from_bytes(sig_data) + result = verify(file=detached_data, store=store, signature=sig) + else: + result = verify(bytes=sig_data, store=store) + except Exception: + raise InvalidSignatureError(_("The signature is not valid.")) + + return VerifyResult(result) def compute_file_hash(filename, hasher=None, cumulative_hash=None, blocksize=8192): diff --git a/pulpcore/content/__init__.py b/pulpcore/content/__init__.py index decf0957b0d..26d5200de45 100644 --- a/pulpcore/content/__init__.py +++ b/pulpcore/content/__init__.py @@ -29,12 +29,18 @@ log = logging.getLogger(__name__) +# PQC (post-quantum) X.509 certificates can exceed aiohttp's default 8190-byte header +# limit when forwarded via X-CLIENT-CERT by a reverse proxy. +_HANDLER_ARGS = {"max_field_size": 16 * 1024} + if settings.OTEL_ENABLED: from .instrumentation import instrumentation # noqa: E402 - app = web.Application(middlewares=[guid, authenticate, instrumentation()]) + app = web.Application( + middlewares=[guid, authenticate, instrumentation()], handler_args=_HANDLER_ARGS + ) else: - app = web.Application(middlewares=[guid, authenticate]) + app = web.Application(middlewares=[guid, authenticate], handler_args=_HANDLER_ARGS) if settings.UVLOOP_ENABLED: diff --git a/pulpcore/exceptions/validation.py b/pulpcore/exceptions/validation.py index 67481723dbd..985aad65dec 100644 --- a/pulpcore/exceptions/validation.py +++ b/pulpcore/exceptions/validation.py @@ -109,7 +109,7 @@ def __str__(self): class InvalidSignatureError(ValidationError): """ - Raised when a signature could not be verified by the GnuPG utility. + Raised when a signature could not be verified. """ error_code = "PLP0021" diff --git a/pulpcore/pytest_plugin.py b/pulpcore/pytest_plugin.py index 35a886ffb0d..2300d992716 100644 --- a/pulpcore/pytest_plugin.py +++ b/pulpcore/pytest_plugin.py @@ -1,7 +1,6 @@ import asyncio import json import os -import pathlib import shutil import socket import ssl @@ -14,7 +13,6 @@ from time import sleep import aiohttp -import gnupg import pytest import requests from aiohttp import web @@ -464,6 +462,200 @@ def ssl_ctx_req_client_auth( return ssl_ctx +# PQC (Post-Quantum Cryptography) TLS Fixtures + +# TODO pretty much all of this can be deleted when "trustme" gets ML-DSA support + + +@dataclass +class PQCCertificateAuthority: + algorithm: str + ca_cert_pem: str + server_cert_path: str + server_key_path: str + client_cert_pem: str + client_key_pem: str + untrusted_client_cert_pem: str + + +_PQC_KEY_CLASSES = { + "ML-DSA-44": "MLDSA44PrivateKey", + "ML-DSA-65": "MLDSA65PrivateKey", + "ML-DSA-87": "MLDSA87PrivateKey", +} + + +def _generate_pqc_certs(td, host, algorithm="ML-DSA-65"): + """Generate a PQC CA and issue server + client certs signed by it.""" + import datetime + import ipaddress + + from cryptography import x509 + from cryptography.hazmat.primitives.asymmetric import mldsa + from cryptography.hazmat.primitives.serialization import Encoding, NoEncryption, PrivateFormat + from cryptography.x509.oid import ExtendedKeyUsageOID, NameOID + + key_cls = getattr(mldsa, _PQC_KEY_CLASSES[algorithm]) + now = datetime.datetime.now(datetime.timezone.utc) + not_before = now - datetime.timedelta(minutes=1) + not_after = now + datetime.timedelta(days=1) + + ca_key = key_cls.generate() + ca_name = x509.Name([x509.NameAttribute(NameOID.COMMON_NAME, "PQC Test CA")]) + ca_cert = ( + x509.CertificateBuilder() + .subject_name(ca_name) + .issuer_name(ca_name) + .public_key(ca_key.public_key()) + .serial_number(x509.random_serial_number()) + .not_valid_before(not_before) + .not_valid_after(not_after) + .add_extension(x509.BasicConstraints(ca=True, path_length=None), critical=True) + .add_extension( + x509.KeyUsage( + digital_signature=True, + content_commitment=False, + key_encipherment=False, + data_encipherment=False, + key_agreement=False, + key_cert_sign=True, + crl_sign=True, + encipher_only=False, + decipher_only=False, + ), + critical=True, + ) + .sign(private_key=ca_key, algorithm=None) + ) + + san = x509.SubjectAlternativeName( + [x509.DNSName("localhost"), x509.IPAddress(ipaddress.ip_address(host))] + ) + + def _issue_cert(name, cn, eku_oid): + key = key_cls.generate() + cert = ( + x509.CertificateBuilder() + .subject_name(x509.Name([x509.NameAttribute(NameOID.COMMON_NAME, cn)])) + .issuer_name(ca_name) + .public_key(key.public_key()) + .serial_number(x509.random_serial_number()) + .not_valid_before(not_before) + .not_valid_after(not_after) + .add_extension(x509.BasicConstraints(ca=False, path_length=None), critical=True) + .add_extension( + x509.KeyUsage( + digital_signature=True, + content_commitment=False, + key_encipherment=False, + data_encipherment=False, + key_agreement=False, + key_cert_sign=False, + crl_sign=False, + encipher_only=False, + decipher_only=False, + ), + critical=True, + ) + .add_extension(x509.ExtendedKeyUsage([eku_oid]), critical=False) + .add_extension(san, critical=False) + .sign(private_key=ca_key, algorithm=None) + ) + cert_pem = cert.public_bytes(Encoding.PEM).decode() + key_pem = key.private_bytes(Encoding.PEM, PrivateFormat.PKCS8, NoEncryption()).decode() + cert_path = str(td / f"{name}.crt") + key_path = str(td / f"{name}.key") + with open(cert_path, "w") as f: + f.write(cert_pem) + with open(key_path, "w") as f: + f.write(key_pem) + return cert_path, key_path, cert_pem, key_pem + + ca_cert_pem = ca_cert.public_bytes(Encoding.PEM).decode() + + server_cert_path, server_key_path, _, _ = _issue_cert( + "server", host, ExtendedKeyUsageOID.SERVER_AUTH + ) + _, _, client_cert_pem, client_key_pem = _issue_cert( + "client", host, ExtendedKeyUsageOID.CLIENT_AUTH + ) + + # Untrusted client: signed by a different CA so cert-chain verification rejects it + untrusted_ca_key = key_cls.generate() + untrusted_ca_name = x509.Name([x509.NameAttribute(NameOID.COMMON_NAME, "PQC Untrusted CA")]) + untrusted_key = key_cls.generate() + untrusted_cert = ( + x509.CertificateBuilder() + .subject_name(x509.Name([x509.NameAttribute(NameOID.COMMON_NAME, "PQC Untrusted Client")])) + .issuer_name(untrusted_ca_name) + .public_key(untrusted_key.public_key()) + .serial_number(x509.random_serial_number()) + .not_valid_before(not_before) + .not_valid_after(not_after) + .add_extension(x509.BasicConstraints(ca=False, path_length=None), critical=True) + .add_extension( + x509.KeyUsage( + digital_signature=True, + content_commitment=False, + key_encipherment=False, + data_encipherment=False, + key_agreement=False, + key_cert_sign=False, + crl_sign=False, + encipher_only=False, + decipher_only=False, + ), + critical=True, + ) + .sign(private_key=untrusted_ca_key, algorithm=None) + ) + untrusted_client_cert_pem = untrusted_cert.public_bytes(Encoding.PEM).decode() + + return PQCCertificateAuthority( + algorithm=algorithm, + ca_cert_pem=ca_cert_pem, + server_cert_path=server_cert_path, + server_key_path=server_key_path, + client_cert_pem=client_cert_pem, + client_key_pem=client_key_pem, + untrusted_client_cert_pem=untrusted_client_cert_pem, + ) + + +@pytest.fixture(scope="session", params=["ML-DSA-65", "ML-DSA-87"]) +def pqc_certificate_authority(request, tmp_path_factory, fixtures_cfg): + """Generate a PQC Certificate Authority and related certs for each ML-DSA variant.""" + algorithm = request.param + td = tmp_path_factory.mktemp(f"pqc_certs_{algorithm}") + return _generate_pqc_certs(td, fixtures_cfg.aiohttp_fixtures_origin, algorithm) + + +@pytest.fixture(scope="session") +def pqc_ssl_ctx(pqc_certificate_authority): + """Server SSL context using PQC certificates (no client auth required).""" + ssl_ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) + ssl_ctx.load_cert_chain( + pqc_certificate_authority.server_cert_path, + pqc_certificate_authority.server_key_path, + ) + return ssl_ctx + + +@pytest.fixture(scope="session") +def pqc_ssl_ctx_req_client_auth(pqc_certificate_authority): + """Server SSL context using PQC certificates, requiring PQC client auth.""" + ssl_ctx = ssl.create_default_context( + purpose=ssl.Purpose.CLIENT_AUTH, + cadata=pqc_certificate_authority.ca_cert_pem, + ) + ssl_ctx.verify_mode = ssl.CERT_REQUIRED + ssl_ctx.load_cert_chain( + pqc_certificate_authority.server_cert_path, + pqc_certificate_authority.server_key_path, + ) + return ssl_ctx + + # Object factories @@ -1060,43 +1252,82 @@ def _dispatch_task_group(task_name, *args, **kwargs): # GPG related fixtures +PULP_FIXTURES_SIGNING_KEYS_URL = ( + "https://raw.githubusercontent.com/pulp/pulp-fixtures/master/common/signing_keys/" +) +KEY_V4_RSA2K_PUBLIC = PULP_FIXTURES_SIGNING_KEYS_URL + "pulp-testkey-v4-rsa2k.asc" +KEY_V4_RSA2K_PRIVATE = PULP_FIXTURES_SIGNING_KEYS_URL + "pulp-testkey-v4-rsa2k.secret" +KEY_V4_RSA4K_PUBLIC = PULP_FIXTURES_SIGNING_KEYS_URL + "pulp-testkey-v4-rsa4k.asc" +KEY_V4_RSA4K_PRIVATE = PULP_FIXTURES_SIGNING_KEYS_URL + "pulp-testkey-v4-rsa4k.secret" +KEY_V4_ED25519_PUBLIC = PULP_FIXTURES_SIGNING_KEYS_URL + "pulp-testkey-v4-ed25519.asc" +KEY_V4_ED25519_PRIVATE = PULP_FIXTURES_SIGNING_KEYS_URL + "pulp-testkey-v4-ed25519.secret" +KEY_V6_RSA4K_PUBLIC = PULP_FIXTURES_SIGNING_KEYS_URL + "pulp-testkey-v6-rsa4k.asc" +KEY_V6_RSA4K_PRIVATE = PULP_FIXTURES_SIGNING_KEYS_URL + "pulp-testkey-v6-rsa4k.secret" +KEY_V6_ED25519_PUBLIC = PULP_FIXTURES_SIGNING_KEYS_URL + "pulp-testkey-v6-ed25519.asc" +KEY_V6_ED25519_PRIVATE = PULP_FIXTURES_SIGNING_KEYS_URL + "pulp-testkey-v6-ed25519.secret" +KEY_V6_MLDSA65_ED25519_PUBLIC = ( + PULP_FIXTURES_SIGNING_KEYS_URL + "pulp-testkey-v6-mldsa65-ed25519.asc" +) +KEY_V6_MLDSA65_ED25519_PRIVATE = ( + PULP_FIXTURES_SIGNING_KEYS_URL + "pulp-testkey-v6-mldsa65-ed25519.secret" +) +KEY_V6_MLDSA87_ED448_PUBLIC = PULP_FIXTURES_SIGNING_KEYS_URL + "pulp-testkey-v6-mldsa87-ed448.asc" +KEY_V6_MLDSA87_ED448_PRIVATE = ( + PULP_FIXTURES_SIGNING_KEYS_URL + "pulp-testkey-v6-mldsa87-ed448.secret" +) + -SIGNING_SCRIPT_STRING = r"""#!/usr/bin/env bash +SIGNING_SCRIPT_STRING = """#!/usr/bin/env bash FILE_PATH=$1 SIGNATURE_PATH="$1.asc" -GPG_KEY_ID="pulp-fixture-signing-key" +GPG_KEY_ID="{gpg_key_id}" # Create a detached signature -gpg --quiet --batch --homedir HOMEDIRHERE --detach-sign --local-user "${GPG_KEY_ID}" \ - --armor --output ${SIGNATURE_PATH} ${FILE_PATH} +gpg --quiet --batch --homedir {gpg_home} --detach-sign --local-user "${{GPG_KEY_ID}}" \\ + --armor --output ${{SIGNATURE_PATH}} ${{FILE_PATH}} # Check the exit status STATUS=$? -if [[ ${STATUS} -eq 0 ]]; then - echo {\"file\": \"${FILE_PATH}\", \"signature\": \"${SIGNATURE_PATH}\"} +if [[ ${{STATUS}} -eq 0 ]]; then + echo '{{"file": "'${{FILE_PATH}}'", "signature": "'${{SIGNATURE_PATH}}'"}}' else - exit ${STATUS} + exit ${{STATUS}} fi """ +SQ_SIGNING_SCRIPT_STRING = """#!/usr/bin/env bash -@pytest.fixture(scope="session") -def signing_script_path(signing_script_temp_dir, signing_gpg_homedir_path): - signing_script_file = signing_script_temp_dir / "sign-metadata.sh" - signing_script_file.write_text( - SIGNING_SCRIPT_STRING.replace("HOMEDIRHERE", str(signing_gpg_homedir_path)) - ) +FILE_PATH=$1 +SIGNATURE_PATH="$1.asc" - signing_script_file.chmod(0o755) +SQ_HOME="{sq_home}" +SIGNER="{signer_fingerprint}" - return signing_script_file +# Create a detached signature using Sequoia (sq) +sq --home "${{SQ_HOME}}" sign --signer "${{SIGNER}}" \\ + --signature-file="${{SIGNATURE_PATH}}" "${{FILE_PATH}}" + +# Check the exit status +STATUS=$? +if [[ ${{STATUS}} -eq 0 ]]; then + echo '{{"file": "'${{FILE_PATH}}'", "signature": "'${{SIGNATURE_PATH}}'"}}' +else + exit ${{STATUS}} +fi +""" + + +@pytest.fixture(scope="session") +def signing_script_path(signing_script_temp_dir, signing_gpg_homedir_path, signing_gpg_metadata): + _gpg, fingerprint, _keyid = signing_gpg_metadata + return make_signing_script(signing_gpg_homedir_path, fingerprint, signing_script_temp_dir) @pytest.fixture(scope="session") def signing_script_temp_dir(tmp_path_factory): - return tmp_path_factory.mktemp("sigining_script_dir") + return tmp_path_factory.mktemp("signing_script_dir") @pytest.fixture(scope="session") @@ -1137,26 +1368,7 @@ def _sign_with_ascii_armored_detached_signing_service(filename): def signing_gpg_metadata(signing_gpg_homedir_path): """A fixture that returns a GPG instance and related metadata (i.e., fingerprint, keyid).""" PRIVATE_KEY_URL = "https://raw.githubusercontent.com/pulp/pulp-fixtures/master/common/GPG-PRIVATE-KEY-fixture-signing" # noqa: E501 - - key_file = pathlib.Path(__file__).parent / "GPG-PRIVATE-KEY-fixture-signing" - if key_file.exists(): - private_key_data = key_file.read_text() - else: - response = requests.get(PRIVATE_KEY_URL) - response.raise_for_status() - private_key_data = response.text - with suppress(FileNotFoundError, PermissionError): - key_file.write_text(private_key_data) - - gpg = gnupg.GPG(gnupghome=signing_gpg_homedir_path) - gpg.import_keys(private_key_data) - - fingerprint = gpg.list_keys()[0]["fingerprint"] - keyid = gpg.list_keys()[0]["keyid"] - - gpg.trust_keys(fingerprint, "TRUST_ULTIMATE") - - return gpg, fingerprint, keyid + return import_signing_key(PRIVATE_KEY_URL, signing_gpg_homedir_path) @pytest.fixture(scope="session") @@ -1178,42 +1390,14 @@ def _ascii_armored_detached_signing_service_name( signing_gpg_metadata, signing_gpg_homedir_path, ): - service_name = str(uuid.uuid4()) - gpg, fingerprint, keyid = signing_gpg_metadata - - cmd = ( - "pulpcore-manager", - "add-signing-service", - service_name, - str(signing_script_path), - fingerprint, - "--class", - "core:AsciiArmoredDetachedSigningService", - "--gnupghome", - str(signing_gpg_homedir_path), - ) - completed_process = subprocess.run( - cmd, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, + _gpg, fingerprint, _keyid = signing_gpg_metadata + service_name = create_signing_service( + signing_gpg_homedir_path, fingerprint, signing_script_path ) - assert completed_process.returncode == 0 - yield service_name - cmd = ( - "pulpcore-manager", - "remove-signing-service", - service_name, - "--class", - "core:AsciiArmoredDetachedSigningService", - ) - subprocess.run( - cmd, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - ) + remove_signing_service(service_name) @pytest.fixture(scope="session") @@ -1225,6 +1409,164 @@ def ascii_armored_detached_signing_service( ).results[0] +@pytest.fixture(scope="session") +def sq_signing_home_path(tmp_path_factory): + return tmp_path_factory.mktemp("sq_home") + + +@pytest.fixture(scope="session") +def sq_signing_script_temp_dir(tmp_path_factory): + return tmp_path_factory.mktemp("sq_signing_script_dir") + + +@pytest.fixture(scope="session") +def sq_signing_metadata(sq_signing_home_path): + """A fixture that returns Sequoia signing metadata (i.e., fingerprint, keyid).""" + return import_signing_key(KEY_V6_ED25519_PRIVATE, sq_signing_home_path, backend="sq") + + +@pytest.fixture(scope="session") +def sq_signing_script_path(sq_signing_script_temp_dir, sq_signing_home_path, sq_signing_metadata): + _sq, fingerprint, _keyid = sq_signing_metadata + return make_signing_script( + sq_signing_home_path, fingerprint, sq_signing_script_temp_dir, backend="sq" + ) + + +@pytest.fixture(scope="session") +def _sq_ascii_armored_detached_signing_service_name( + sq_signing_script_path, + sq_signing_metadata, + sq_signing_home_path, +): + _sq, fingerprint, _keyid = sq_signing_metadata + service_name = create_signing_service( + sq_signing_home_path, fingerprint, sq_signing_script_path, backend="sq" + ) + + yield service_name + + remove_signing_service(service_name) + + +@pytest.fixture(scope="session") +def sq_ascii_armored_detached_signing_service( + _sq_ascii_armored_detached_signing_service_name, pulpcore_bindings +): + return pulpcore_bindings.SigningServicesApi.list( + name=_sq_ascii_armored_detached_signing_service_name + ).results[0] + + +def import_signing_key(key_url, home, *, backend="gpg"): + """Import a PGP key into a keyring and return metadata. + + Returns `(gpg_instance_or_none, fingerprint, keyid)`. The first element + is a `gnupg.GPG` instance when `backend` is `"gpg"`, or `None` when + `backend` is `"sq"`. + """ + response = requests.get(key_url) + response.raise_for_status() + + if backend == "sq": + from pysequoia import Cert + + completed = subprocess.run( + ("sq", "--home", str(home), "key", "import"), + input=response.content, + capture_output=True, + ) + assert completed.returncode == 0, completed.stderr.decode() + + cert = Cert.from_bytes(response.content) + fingerprint = cert.fingerprint.upper() + keyid = fingerprint[-16:] + + return None, fingerprint, keyid + else: + try: + import gnupg + except ImportError: + pytest.skip("python-gnupg not installed") + + gpg = gnupg.GPG(gnupghome=home) + + result = gpg.import_keys(response.content) + assert result.count >= 1, f"Failed to import key from {key_url}" + + key_info = gpg.list_keys()[0] + fingerprint = key_info["fingerprint"] + keyid = key_info["keyid"] + gpg.trust_keys(fingerprint, "TRUST_ULTIMATE") + + return gpg, fingerprint, keyid + + +def make_signing_script(home, fingerprint, script_dir=None, *, backend="gpg"): + """Create a detached-signature signing script. + + Returns the script path. + """ + if script_dir is None: + script_dir = home + if backend == "sq": + script_path = script_dir / "sq_sign.sh" + script_path.write_text( + SQ_SIGNING_SCRIPT_STRING.format(sq_home=home, signer_fingerprint=fingerprint) + ) + else: + script_path = script_dir / "sign.sh" + script_path.write_text(SIGNING_SCRIPT_STRING.format(gpg_home=home, gpg_key_id=fingerprint)) + script_path.chmod(0o755) + return script_path + + +def create_signing_service( + home, + fingerprint, + script_path, + *, + backend="gpg", + service_class="core:AsciiArmoredDetachedSigningService", +): + """Register a signing service via pulpcore-manager. + + Returns the service name. + """ + service_name = str(uuid.uuid4()) + cmd = [ + "pulpcore-manager", + "add-signing-service", + service_name, + str(script_path), + fingerprint, + "--class", + service_class, + "--backend", + backend, + "--home", + str(home), + ] + completed = subprocess.run(cmd, capture_output=True, text=True) + assert completed.returncode == 0, completed.stderr + + return service_name + + +def remove_signing_service(service_name, service_class="core:AsciiArmoredDetachedSigningService"): + """Remove a signing service created by ``create_signing_service``.""" + subprocess.run( + ( + "pulpcore-manager", + "remove-signing-service", + service_name, + "--class", + service_class, + ), + capture_output=True, + ) + + # if content_origin == None, base_url will return the relative path and # we need to add the hostname to run the tests @pytest.fixture(scope="session") diff --git a/pulpcore/tests/functional/api/test_signing_service.py b/pulpcore/tests/functional/api/test_signing_service.py index 25f4a477d40..717b32fe63b 100644 --- a/pulpcore/tests/functional/api/test_signing_service.py +++ b/pulpcore/tests/functional/api/test_signing_service.py @@ -1,7 +1,43 @@ import pytest +from pulpcore.pytest_plugin import ( + KEY_V4_RSA4K_PRIVATE, + create_signing_service, + import_signing_key, + make_signing_script, + remove_signing_service, +) + @pytest.mark.parallel -def test_crud_signing_service(ascii_armored_detached_signing_service): - service = ascii_armored_detached_signing_service +@pytest.mark.parametrize( + "signing_service_fixture", + [ + "ascii_armored_detached_signing_service", + "sq_ascii_armored_detached_signing_service", + ], +) +def test_crud_signing_service(signing_service_fixture, request): + service = request.getfixturevalue(signing_service_fixture) assert "/api/v3/signing-services/" in service.pulp_href + + +@pytest.mark.parametrize("backend", ["gpg", "sq"]) +def test_add_signing_service_key_with_subkeys(backend, tmp_path_factory): + """Verify that add-signing-service works with a PGP key that has subkeys. + + Keys with signing subkeys produce multiple fpr: lines in GPG's colon + output, which previously caused add-signing-service to fail. + + With both GPG and Sequoia backends, the service should be created + successfully with the primary key fingerprint. + """ + home = tmp_path_factory.mktemp(f"{backend}_subkey_test") + script_dir = tmp_path_factory.mktemp(f"{backend}_subkey_script") + _gpg, fingerprint, _keyid = import_signing_key(KEY_V4_RSA4K_PRIVATE, home, backend=backend) + script_path = make_signing_script(home, fingerprint, script_dir, backend=backend) + service_name = create_signing_service(home, fingerprint, script_path, backend=backend) + + assert len(fingerprint) in (40, 64) + + remove_signing_service(service_name) diff --git a/pulpcore/tests/functional/api/using_plugin/test_content_delivery.py b/pulpcore/tests/functional/api/using_plugin/test_content_delivery.py index 0961e4f5350..c398be01372 100644 --- a/pulpcore/tests/functional/api/using_plugin/test_content_delivery.py +++ b/pulpcore/tests/functional/api/using_plugin/test_content_delivery.py @@ -154,7 +154,7 @@ def test_remote_content_changed_with_on_demand( # THEN assert not output_file.exists() assert result.returncode == 18 - assert b"* Closing connection 0" in result.stderr + assert b"closing connection" in result.stderr.lower() assert b"curl: (18) transfer closed with outstanding read data remaining" in result.stderr # WHEN (second request) diff --git a/pyproject.toml b/pyproject.toml index 56dcb291bbb..847b378b609 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -31,7 +31,7 @@ dependencies = [ "asyncio-throttle>=1.0,<=1.0.2", # Unsure about versioning, but not released often anyway. "backoff>=2.1.2,<2.3", # Looks like only bugfixes in z-Stream. "click>=8.1.0,<8.4", # Uses milestone.feature.fix https://palletsprojects.com/versions . - "cryptography>=44.0.3,<47.0", # SemVer compatible https://cryptography.io/en/latest/api-stability/#versioning . + "cryptography>=49.0.0,<51.0", # SemVer compatible https://cryptography.io/en/latest/api-stability/#versioning . "Django>=4.2.24,<5.3, !=5.0, !=5.1", # LTS version, switch only if we have a compelling reason to". "django-filter>=23.1,<=25.2", # Uses CalVer, not released often https://github.com/carltongibson/django-filter "django-guid>=3.3.0,<3.6", # Looks like only bugfixes in z-Stream. @@ -48,16 +48,16 @@ dependencies = [ "jinja2>=3.1,<=3.1.6", "json_stream>=2.3.2,<2.5", "jq>=1.6.0,<1.12.0", - "PyOpenSSL<27.0", + "PyOpenSSL>=26.3.0,<27.0", "opentelemetry-api>=1.27.0,<1.41", "opentelemetry-sdk>=1.27.0,<1.41", "opentelemetry-exporter-otlp-proto-http>=1.27.0,<1.41", "protobuf>=4.21.1,<7.0", "pulp-glue>=0.30.0,<0.40", "pygtrie>=2.5,<=2.5.0", - "psycopg[binary]>=3.1.8,<3.4", # SemVer, not explicitely stated, but mentioned on multiple changes. + "psycopg[binary]>=3.3.4,<3.4", # SemVer, not explicitely stated, but mentioned on multiple changes. "pyparsing>=3.1.0,<3.4", # Looks like only bugfixes in z-Stream. - "python-gnupg>=0.5.0,<0.6", # Looks like only bugfixes in z-Stream [changelog only in git] + "pysequoia>=0.1.33,<0.2", "PyYAML>=5.1.1,<6.1", # Looks like only bugfixes in z-Stream. "redis>=4.3.0,<7.2", # Looks like only bugfixes in z-Stream. "tablib>=3.5.0,<4.0, !=3.6", # 3.6.0 breaks with import export. Not sure about semver. diff --git a/template_config.yml b/template_config.yml index 98619948b6b..507b2bdcc1b 100644 --- a/template_config.yml +++ b/template_config.yml @@ -6,10 +6,13 @@ # After editing this file please always reapply the plugin template before committing any changes. --- +# NOTE: the S3 runner has redis disabled, domains enabled, and hide_guarded_distributions = true +# "weird" test failures unique to that runner may be due to those differences. + check_commit_message: true check_manifest: true check_stray_pulpcore_imports: false -ci_base_image: "ghcr.io/pulp/pulp-ci-centos9" +ci_base_image: "ghcr.io/pulp/pulp-ci-centos10" ci_env: {} ci_trigger: "{pull_request: {branches: ['*']}}" cli_package: "pulp-cli"