From 8980046313434a01dc151c8438bdab075074cfc4 Mon Sep 17 00:00:00 2001 From: Daniel Alley Date: Mon, 9 Mar 2026 16:55:18 -0400 Subject: [PATCH 01/10] Replace de-novo OpenPGP parsing code with pysequoia Generated-By: claude-opus-4.6 (cherry picked from commit 9a1bde65b6808b0f18aca7b50cdcce015fb60b48) --- pulpcore/app/models/openpgp.py | 14 +- pulpcore/app/openpgp.py | 582 ++++----------------------------- pyproject.toml | 1 + 3 files changed, 76 insertions(+), 521 deletions(-) diff --git a/pulpcore/app/models/openpgp.py b/pulpcore/app/models/openpgp.py index b6d2b8eaf62..8f6a11c3062 100644 --- a/pulpcore/app/models/openpgp.py +++ b/pulpcore/app/models/openpgp.py @@ -3,9 +3,9 @@ from aiohttp.web_response import Response from django.db import models from django.utils import timezone +from pysequoia import armor, ArmorKind from pulpcore.app.models import AutoAddObjPermsMixin, Content, Distribution, Repository -from pulpcore.app.openpgp import wrap_armor from pulpcore.app.util import get_domain_pk, gpg_verify @@ -62,7 +62,7 @@ def represent(self, repository_version=None): data += public_subkey.packet() for signature in public_subkey.openpgp_signatures.filter(**content_filter): data += signature.packet() - return wrap_armor(data) + return armor(data, ArmorKind.PublicKey).strip() # avoid trailing newline class Meta: default_related_name = "%(app_label)s_%(model_name)s" @@ -128,11 +128,11 @@ class OpenPGPSignature(_OpenPGPContent): sha256 = models.CharField(max_length=128) signature_type = models.PositiveSmallIntegerField() - created = models.DateTimeField() # 2 - expiration_time = models.DurationField(null=True) # 3 - key_expiration_time = models.DurationField(null=True) # 9 - issuer = models.CharField(max_length=16, null=True) # 16 - signers_user_id = models.CharField(null=True) # 28 + created = models.DateTimeField() + expiration_time = models.DurationField(null=True) + key_expiration_time = models.DurationField(null=True) + issuer = models.CharField(max_length=16, null=True) + signers_user_id = models.CharField(null=True) signed_content = models.ForeignKey( Content, related_name="openpgp_signatures", on_delete=models.PROTECT ) diff --git a/pulpcore/app/openpgp.py b/pulpcore/app/openpgp.py index f25639ce5b0..7335970bd99 100644 --- a/pulpcore/app/openpgp.py +++ b/pulpcore/app/openpgp.py @@ -1,529 +1,83 @@ import hashlib -from base64 import b64decode, b64encode -from django.utils import timezone - -# Source of information: -# * rfc4880 -# * rfc4880bis -# * gnupg sources -# * https://datatracker.ietf.org/doc/html/draft-shaw-openpgp-hkp-00 - - -PACKET_TYPES = { - 1: "Public-Key Encrypted Session Key Packet", - 2: "Signature Packet", - 3: "Symmetric-Key Encrypted Session Key Packet", - 4: "One-Pass Signature Packet", - 5: "Secret-Key Packet", - 6: "Public-Key Packet", - 7: "Secret-Subkey Packet", - 8: "Compressed Data Packet", - 9: "Symmetrically Encrypted Data Packet", - 10: "Marker Packet", - 11: "Literal Data Packet", - 12: "Trust Packet", - 13: "User ID Packet", - 14: "Public-Subkey Packet", - 17: "User Attribute Packet", - 18: "Sym. Encrypted and Integrity Protected Data Packet", - 19: "Modification Detection Code Packet", - 20: "OCB Encrypted Data Packet", -} - - -SIG_SUBPACKAGE_TYPES = { - 2: "Signature Creation Time", - 3: "Signature Expiration Time", - 4: "Exportable Certification", - 5: "Trust Signature", - 6: "Regular Expression", - 7: "Revocable", - 9: "Key Expiration Time", - 10: "Placeholder for backward compatibility", - 11: "Preferred Symmetric Algorithms", - 12: "Revocation Key", - 16: "Issuer", - 20: "Notation Data", - 21: "Preferred Hash Algorithms", - 22: "Preferred Compression Algorithms", - 23: "Key Server Preferences", - 24: "Preferred Key Server", - 25: "Primary User ID", - 26: "Policy URI", - 27: "Key Flags", - 28: "Signer's User ID", - 29: "Reason for Revocation", - 30: "Features", - 31: "Signature Target", - 32: "Embedded Signature", - 33: "Issuer Fingerprint", - 34: "Preferred Encryption Modes", - 35: "Intended Recipient Fingerprint", - 37: "Attested Certifications", - 38: "Key Block", - 40: "Literal Data Meta Hash", - 41: "Trust Alias", -} - - -PUBKEY_ALGORITHMS = { - 1: { - "name": "RSA", - "format": "mm", - }, - 2: { - "name": "RSA Encrypt-Only", - "format": "mm", - }, - 3: { - "name": "RSA Sign-Only", - "format": "mm", - }, - 16: { - "name": "Elgamal", - "format": "mmm", - }, - 17: { - "name": "DSA", - "format": "mmmm", - }, - 18: { - "name": "ECDH", - "format": "omk", - }, - 19: { - "name": "ECDSA", - "format": "om", - }, - 22: { - "name": "EdDSA", - "format": "om", - }, -} - - -HASH_ALGORITHMS = { - 1: "md5", - 2: "sha1", - 3: "ripemd160", - 8: "sha256", - 9: "sha384", - 10: "sha512", - 11: "sha224", - 12: "sha3-256", - 14: "sha3-512", -} - - -SYMMETRIC_ALGORITHMS = { - 0: "Plaintext", - 1: "IDEA", - 2: "TripleDES", - 3: "CAST5", - 4: "Blowfish", - 7: "AES128", - 8: "AES192", - 9: "AES256", - 10: "Twofish", - 11: "Camellia 128", - 12: "Camellia 192", - 13: "Camellia 256", -} - - -COMPRESSION_ALGORITHMS = { - 0: "Uncompressed", - 1: "ZIP", - 2: "ZLIB", - 3: "BZip2", -} - - -ENCRYPTION_MODES = { - 1: "EAX", - 2: "OCB", -} - - -def packet_iter(data): - begin = 0 - pos = 0 - while pos < len(data): - packet_tag = data[pos] - pos += 1 - if not packet_tag & 0x80: - raise ValueError("Invalid Packet Tag") - new_format = bool(packet_tag & 0x40) - - if new_format: - packet_type = packet_tag & 0x1F - - if data[pos] < 0xC0: - # 1-octet length - length = data[pos] - pos += 1 - elif data[pos] < 0xE0: - # 2-octet length - length = ((data[pos] - 0xC0) << 8) + data[pos + 1] + 0xC0 - pos += 2 - elif data[pos] == 0xFF: - # 5-octet length - length = int.from_bytes(data[pos + 1 : pos + 5], "big") - pos += 5 - else: - # Partial body length - raise NotImplementedError("Partial Packet Body is not implemented") - else: - packet_type = (packet_tag & 0x3C) >> 2 - length_type = packet_tag & 0x03 - if length_type == 3: - # Indeterminate packet length - length = len(data) - pos - else: - length_bytes = 1 << length_type - length = int.from_bytes(data[pos : pos + length_bytes], "big") - pos += length_bytes - - if packet_type == 0: - raise ValueError("Invalid Packet Type") - yield { - "type": packet_type, - "body": data[pos : pos + length], - "raw": data[begin : pos + length], - } - pos += length - begin = pos - if pos != len(data): - raise ValueError("Broken Stream") - - -def subpacket_iter(data): - begin = 0 - pos = 0 - while pos < len(data): - if data[pos] < 0xC0: - # 1-octet length - length = data[pos] - pos += 1 - elif data[pos] < 0xD0: - # 2-octet length - length = ((data[pos] - 0xC0) << 8) + data[pos + 1] + 0xC0 - pos += 2 - elif data[pos] == 0xFF: - # 5-octet length - length = int.from_bytes(data[pos + 1 : pos + 5], "big") - pos += 5 - else: - raise ValueError("Partial packet lengths are not allowed.") - yield { - "type": data[pos] & 0x7F, - "critical": bool(data[pos] & 0x80), - "body": data[pos + 1 : pos + length], - "raw": data[begin : pos + length], - } - pos += length - begin = pos - if pos != len(data): - raise ValueError("Broken Stream") - - -def extract_mpi(data): - bit_length = int.from_bytes(data[0 : 0 + 2], "big") - length = (bit_length + 7) // 8 - return {"bit_length": bit_length, "body": data[2 : 2 + length], "raw": data[0 : 2 + length]} - - -def extract_oid_kdf(data): - # These two types use the same method to decribe the length, which is all we want. - length = data[0] - return {"body": data[1 : 1 + length], "raw": data[0 : 1 + length]} - - -def analyze_sig_subpackets(data): - signature_attributes = {} - for packet in subpacket_iter(data): - packet_type = packet["type"] - body = packet["body"] - if packet_type == 2: - signature_attributes["created"] = timezone.datetime.fromtimestamp( - int.from_bytes(body, "big") - ).astimezone() - elif packet_type == 3: - signature_attributes["expiration_time"] = timezone.timedelta( - seconds=int.from_bytes(body, "big") - ) - elif packet_type == 9: - signature_attributes["key_expiration_time"] = timezone.timedelta( - seconds=int.from_bytes(body, "big") - ) - elif packet_type == 16: - signature_attributes["issuer"] = body.hex() - elif packet_type == 28: - signature_attributes["signers_user_id"] = body.decode() - return signature_attributes - - -def analyze_signature(data, pubkey, signed_packet_type, signed_packet): - # Type 2 - version = data[0] - if version == 3: - raise NotImplementedError("Version 3 signatures are not implemented.") - elif version in [4, 5]: - signature_type = data[1] - # pubkey_algorithm = PUBKEY_ALGORITHMS.get(data[2]) # Unused here. - hash_algorithm = HASH_ALGORITHMS.get(data[3]) - hashed_size = (data[4] << 8) + data[5] - hashed_data = data[6 : 6 + hashed_size] - unhashed_size = (data[6 + hashed_size] << 8) + data[7 + hashed_size] - unhashed_data = data[8 + hashed_size : 8 + hashed_size + unhashed_size] - canary = data[8 + hashed_size + unhashed_size : 10 + hashed_size + unhashed_size] - # signature = data[10 + hashed_size + unhashed_size :] # Unused here. - - if signature_type in [0x18, 0x19, 0x28]: - # 0x18 Subkey Binding Signature - # 0x19 Primary Key Binding Signature - # 0x28 Subkey Revocation Signature - if signed_packet_type != 14: - raise ValueError("Out of band subkey key signature.") - if version == 4: - hash_payload = b"\x99" + len(signed_packet).to_bytes(2, "big") + signed_packet - else: # version == 5 - hash_payload = b"\x9a" + len(signed_packet).to_bytes(4, "big") + signed_packet - elif signature_type in [0x10, 0x11, 0x12, 0x13, 0x16, 0x30]: - # 0x10 - 0x13 Certification of a user id or attribute - # 0x16 Attested Key Signature - # 0x30 Certification Revocation Signature - if signed_packet_type == 13: - hash_payload = b"\xb4" + len(signed_packet).to_bytes(4, "big") + signed_packet - elif signed_packet_type == 17: - hash_payload = b"\xd1" + len(signed_packet).to_bytes(4, "big") + signed_packet - else: - raise ValueError("Out of band user ID or attribute signature.") - elif signature_type in [0x1F, 0x20, 0x30]: - # 0x1F Direct Key Signature - # 0x20 Key Revocation Signature - # 0x30 Certification Revocation Signature - if signed_packet_type != 6: - raise ValueError("Out of band key signature.") - hash_payload = b"" - else: - # 0x50 Third-Party Confirmation Signature (does this even apply to keys?) - raise NotImplementedError(f"Unsupported signature type {signature_type:#x}.") - - # Validate the signature against the canary value - if hash_algorithm is None: - raise ValueError(f"Unknown hash algorithm {data[3]:#x} used for signature.") - h = hashlib.new(hash_algorithm) - if version == 4: - h.update(b"\x99" + len(pubkey).to_bytes(2, "big") + pubkey) - else: # version == 5 - h.update(b"\x9a" + len(pubkey).to_bytes(4, "big") + pubkey) - h.update(hash_payload) - if version == 4: - h.update( - data[: 6 + hashed_size] - + b"\x04\xff" - + ((6 + hashed_size) % (1 << 32)).to_bytes(4, "big") - ) - else: # version == 5 - h.update( - data[: 6 + hashed_size] - + b"\x05\xff" - + ((6 + hashed_size) % (1 << 64)).to_bytes(8, "big") - ) - if not h.digest().startswith(canary): - raise ValueError("Signature canary mismatch") - - # Hash the signature packet for db-uniqueness - sha256 = hashlib.sha256(data).hexdigest() - signature_attributes = { - "sha256": sha256, - "signature_type": signature_type, - "raw_data": data, - } - # Hashed Subpackets - signature_attributes.update(analyze_sig_subpackets(hashed_data)) - # Unhashed Subpackets - signature_attributes.update(analyze_sig_subpackets(unhashed_data)) - return signature_attributes - else: - raise ValueError(f"Invalid Packet version {version}") - - -def analyze_user_id(data): - # Type 13 - user_id = data.decode() - return {"raw_data": data, "user_id": user_id} - - -def analyze_user_attribute(data): - # Type 17 - # Treat as an opaque packet for now. - sha256 = hashlib.sha256(data).hexdigest() - return {"raw_data": data, "sha256": sha256} - - -def analyze_pubkey(data): - # Type 5, 6, 7 or 14 - # Type 5 and 7 are actually secret key packages. They begin with the corresponding public key - # package. Secret bits are ignored by us here. - version = data[0] - created = timezone.datetime.fromtimestamp(int.from_bytes(data[1:5], "big")).astimezone() - if version == 3: - n = extract_mpi(data[8:]) - e = extract_mpi(data[8 + len(n["raw"])]) - fingerprint = hashlib.md5(n["body"] + e["body"]).hexdigest() - # expiration = int.from_bytes(data[5:7], "big") # Unused here. Kept for documentation. - pubkey_algorithm = PUBKEY_ALGORITHMS.get(data[7]) - elif version in [4, 5]: - pubkey_algorithm = PUBKEY_ALGORITHMS.get(data[5]) - if version == 4: - key_data = data[6:] - else: - key_data_len = int.from_bytes(data[6:10], "big") - key_data = data[10 : 10 + key_data_len] - pub_key_body = data[: 10 + key_data_len] - fingerprint = hashlib.sha256( - b"\x9a" + len(pub_key_body).to_bytes(4, "big") + pub_key_body - ).hexdigest() - if pubkey_algorithm and "format" in pubkey_algorithm: - pos = 0 - for item_type in pubkey_algorithm["format"]: - if item_type == "m": - # Multi precision integer - mpi = extract_mpi(key_data[pos:]) - pos += len(mpi["raw"]) - elif item_type == "o": - # OID - oid = extract_oid_kdf(key_data[pos:]) - pos += len(oid["raw"]) - elif item_type == "k": - # KDF parameters - kdf = extract_oid_kdf(key_data[pos:]) - pos += len(kdf["raw"]) - else: - raise RuntimeError("Unknown key material format.") - if version == 4: - key_data_len = pos - pub_key_body = data[: 6 + key_data_len] - fingerprint = hashlib.sha1( - b"\x99" + len(pub_key_body).to_bytes(2, "big") + pub_key_body - ).hexdigest() - else: - if version == 4: - # We needed to analyse the public key algorithm to calculate the fingerprint of - # version 4 keys. Version 5 keys do not have this limitation, and we can get away - # with an unknown algorithm. - raise ValueError("Unknown public key algorithm.") - else: - raise ValueError(f"Invalid Packet version {version}") - return {"raw_data": data, "created": created, "fingerprint": fingerprint} - - -def gpg_crc24(data): - crc = 0xB704CE - for byte in data: - crc ^= byte << 16 - for i in range(8): - crc <<= 1 - if crc & 0x1000000: - crc ^= 0x1864CFB - return (crc & 0xFFFFFF).to_bytes(3, "big") - - -def unwrap_armor(data): - try: - lines = data.decode().strip().split("\n") - except UnicodeDecodeError: - # assume raw binary data - return data - line = lines.pop(0).strip() - if line.startswith("-----BEGIN ") and line.endswith("-----"): - message_type = line[11:-5] - else: - # Header not found assume raw binary data - return data - line = lines.pop(0).strip() - while line != "": - # Armor Headers - line = lines.pop(0).strip() - armor = "" - while line != "-----END " + message_type + "-----": - armor += line - line = lines.pop(0).strip() - if armor[-5] != "=": - raise ValueError("Broken Stream") - raw = b64decode(armor[:-5]) - checksum = b64decode(armor[-4:]) - if gpg_crc24(raw) != checksum: - raise ValueError("Checksum Mismatch") - return raw - - -def wrap_armor(raw, message_type="PGP PUBLIC KEY BLOCK"): - checksum = "=" + b64encode(gpg_crc24(raw)).decode() - data = b64encode(raw).decode() - lines = ["-----BEGIN " + message_type + "-----", ""] - while data: - line = data[:76] - data = data[76:] - lines.append(line) - lines.append(checksum) - lines.append("-----END " + message_type + "-----") - return "\n".join(lines) +from pysequoia.packet import PacketPile, Tag def read_public_key(data): - data = unwrap_armor(data) - packets = packet_iter(data) - - # The first packet must be the public key. - packet = next(packets) - if packet["type"] != 6: - raise ValueError("Not a public key.") - public_key = analyze_pubkey(packet["body"]) - public_key.update( - {"user_ids": [], "user_attributes": [], "public_subkeys": [], "signatures": []} - ) - signed_content = public_key - signed_packet_type = 6 - - for packet in packets: - packet_type = packet["type"] - body = packet["body"] - if packet_type == 2: - signed_content["signatures"].append( - analyze_signature( - body, - public_key["raw_data"], - signed_packet_type, - signed_content["raw_data"], - ) - ) + pile = PacketPile.from_bytes(data) + + public_key = None + signed_content = None + + for packet in pile: + tag = packet.tag + body = bytes(packet.body) + + if tag == Tag.PublicKey: + if public_key is not None: + raise ValueError("Multiple public keys found.") + public_key = { + "raw_data": body, + "fingerprint": packet.fingerprint, + "created": packet.key_created, + "user_ids": [], + "user_attributes": [], + "public_subkeys": [], + "signatures": [], + } + signed_content = public_key + + elif tag == Tag.PublicSubkey: + public_subkey = { + "raw_data": body, + "fingerprint": packet.fingerprint, + "created": packet.key_created, + "signatures": [], + } + signed_content = public_subkey + public_key["public_subkeys"].append(public_subkey) - elif packet_type == 13: - user_id = analyze_user_id(body) - user_id["signatures"] = [] + elif tag == Tag.UserID: + user_id = { + "raw_data": body, + "user_id": packet.user_id, + "signatures": [], + } signed_content = user_id - signed_packet_type = packet_type public_key["user_ids"].append(user_id) - elif packet_type == 14: - public_subkey = analyze_pubkey(body) - public_subkey["signatures"] = [] - signed_content = public_subkey - signed_packet_type = packet_type - public_key["public_subkeys"].append(public_subkey) - - elif packet_type == 17: - user_attribute = analyze_user_attribute(body) - user_attribute["signatures"] = [] + elif tag == Tag.UserAttribute: + user_attribute = { + "raw_data": body, + "sha256": hashlib.sha256(body).hexdigest(), + "signatures": [], + } signed_content = user_attribute - signed_packet_type = packet_type public_key["user_attributes"].append(user_attribute) + elif tag == Tag.Signature: + sig_attrs = { + "sha256": hashlib.sha256(body).hexdigest(), + "signature_type": body[1], + "raw_data": body, + "created": packet.signature_created, + } + # Note: Pulp's concept of "expiration time" is a duration starting at creation time + # Sequoia calls that "validity_period", while "expiration_time" is an instant + if packet.signature_validity_period is not None: + sig_attrs["expiration_time"] = packet.signature_validity_period + if packet.key_validity_period is not None: + sig_attrs["key_expiration_time"] = packet.key_validity_period + if packet.issuer_key_id is not None: + sig_attrs["issuer"] = packet.issuer_key_id + if packet.signers_user_id is not None: + sig_attrs["signers_user_id"] = packet.signers_user_id + signed_content["signatures"].append(sig_attrs) + else: - raise NotImplementedError("Invalid or unknown rfc4880 packet.") + raise NotImplementedError(f"Unexpected packet type: {tag!r}") + + if public_key is None: + raise ValueError("Not a public key.") return public_key diff --git a/pyproject.toml b/pyproject.toml index 56dcb291bbb..ed0d81c9498 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -58,6 +58,7 @@ dependencies = [ "psycopg[binary]>=3.1.8,<3.4", # SemVer, not explicitely stated, but mentioned on multiple changes. "pyparsing>=3.1.0,<3.4", # Looks like only bugfixes in z-Stream. "python-gnupg>=0.5.0,<0.6", # Looks like only bugfixes in z-Stream [changelog only in git] + "pysequoia==0.1.32", "PyYAML>=5.1.1,<6.1", # Looks like only bugfixes in z-Stream. "redis>=4.3.0,<7.2", # Looks like only bugfixes in z-Stream. "tablib>=3.5.0,<4.0, !=3.6", # 3.6.0 breaks with import export. Not sure about semver. From c9ecf0ec01ac5ce01c3bdc27fd433f49842ea4b8 Mon Sep 17 00:00:00 2001 From: Daniel Alley Date: Mon, 9 Mar 2026 17:02:26 -0400 Subject: [PATCH 02/10] Remove python-gnupg dependency Use pysequoia instead, or shell out to gpg on the command line, which is what python-gnupg does anyway. It's not much additional code to just ditch the dependency everywhere. Assisted-By: claude-opus-4.6 (cherry picked from commit bc6396aa26d65b1d58ec32ed604f96a192064483) --- functest_requirements.txt | 2 +- .../commands/add-signing-service.py | 34 +++++++-- pulpcore/app/models/openpgp.py | 3 +- pulpcore/app/util.py | 73 ++++++++++++++----- pulpcore/exceptions/validation.py | 2 +- pulpcore/pytest_plugin.py | 45 ++++++++++-- pyproject.toml | 3 +- 7 files changed, 124 insertions(+), 38 deletions(-) diff --git a/functest_requirements.txt b/functest_requirements.txt index c8bb7e387ce..dce5f79b0db 100644 --- a/functest_requirements.txt +++ b/functest_requirements.txt @@ -1,7 +1,7 @@ pytest<10 pytest-custom_exit_code pytest-xdist -python-gnupg +pysequoia proxy.py~=2.4.10 trustme~=1.2.1 diff --git a/pulpcore/app/management/commands/add-signing-service.py b/pulpcore/app/management/commands/add-signing-service.py index 9cb4f1b32b5..5c463da49c0 100644 --- a/pulpcore/app/management/commands/add-signing-service.py +++ b/pulpcore/app/management/commands/add-signing-service.py @@ -1,8 +1,8 @@ import os +import subprocess from gettext import gettext as _ from pathlib import Path -import gnupg from django.apps import apps from django.core.management import BaseCommand, CommandError from django.db.utils import IntegrityError @@ -68,13 +68,33 @@ def handle(self, *args, **options): ) ) - gpg = gnupg.GPG(gnupghome=options["gnupghome"], keyring=options["keyring"]) + gpg_cmd = ["gpg"] + if options["gnupghome"]: + gpg_cmd += ["--homedir", options["gnupghome"]] + if options["keyring"]: + gpg_cmd += ["--keyring", options["keyring"]] - key_list = gpg.list_keys(keys=[key_id]) - if not len(key_list) == 1: - raise CommandError(_("There are {} keys matching the key id.").format(len(key_list))) - fingerprint = key_list[0]["fingerprint"] - public_key = gpg.export_keys(key_id) + result = subprocess.run( + gpg_cmd + ["--with-colons", "--fingerprint", key_id], + capture_output=True, + text=True, + ) + if result.returncode != 0: + raise CommandError(result.stderr.strip()) + + fpr_lines = [line for line in result.stdout.splitlines() if line.startswith("fpr:")] + if len(fpr_lines) != 1: + raise CommandError(_("There are {} keys matching the key id.").format(len(fpr_lines))) + fingerprint = fpr_lines[0].split(":")[9] + + result = subprocess.run( + gpg_cmd + ["--armor", "--export", key_id], + capture_output=True, + text=True, + ) + if result.returncode != 0: + raise CommandError(result.stderr.strip()) + public_key = result.stdout try: script_path = Path(script).resolve(strict=True) diff --git a/pulpcore/app/models/openpgp.py b/pulpcore/app/models/openpgp.py index 8f6a11c3062..3c96e03e8f0 100644 --- a/pulpcore/app/models/openpgp.py +++ b/pulpcore/app/models/openpgp.py @@ -3,7 +3,7 @@ from aiohttp.web_response import Response from django.db import models from django.utils import timezone -from pysequoia import armor, ArmorKind +from pysequoia import ArmorKind, armor from pulpcore.app.models import AutoAddObjPermsMixin, Content, Distribution, Repository from pulpcore.app.util import get_domain_pk, gpg_verify @@ -48,6 +48,7 @@ def represent(self, repository_version=None): else: content_filter = {} data = self.packet() + # note: because these queries aren't ordered, the result may be nondetermininistic for signature in self.openpgp_signatures.filter(**content_filter): data += signature.packet() for user_id in self.user_ids.filter(**content_filter): diff --git a/pulpcore/app/util.py b/pulpcore/app/util.py index 8ee0a1a66db..2b0a51f2f82 100644 --- a/pulpcore/app/util.py +++ b/pulpcore/app/util.py @@ -1,7 +1,6 @@ import hashlib import os import socket -import tempfile import zlib from contextlib import ExitStack from datetime import timedelta @@ -14,7 +13,6 @@ from urllib.parse import urlparse from uuid import UUID -import gnupg from django.apps import apps from django.conf import settings from django.db import connection @@ -383,9 +381,36 @@ def get_request_without_query_params(context): return request +class VerifyResult: + """ + Verification result mimicking the interface of gnupg.Verify for compatibility. + + Attributes: + valid (bool): Always True; invalid signatures raise InvalidSignatureError instead. + fingerprint (str): Fingerprint of the signing subkey (uppercase hex). + pubkey_fingerprint (str): Fingerprint of the signing certificate (uppercase hex). + key_id (str): Short (16-char) key ID derived from the fingerprint. + data (bytes or None): The verified plaintext content for inline signatures, None for + detached signatures. + """ + + def __init__(self, decrypted): + self.valid = True + self.data = bytes(decrypted.bytes) if decrypted.bytes is not None else None + vs = decrypted.valid_sigs[0] + self.fingerprint = vs.signing_key.upper() + self.pubkey_fingerprint = vs.certificate.upper() + self.key_id = vs.signing_key[-16:].upper() + + def __repr__(self): + return ( + f"" + ) + + def gpg_verify(public_keys, signature, detached_data=None): """ - Check whether the provided gnupg signature is valid for one of the provided public keys. + Check whether the provided signature is valid for one of the provided public keys. Args: public_keys (str): Ascii armored public key data @@ -394,25 +419,37 @@ def gpg_verify(public_keys, signature, detached_data=None): signature Returns: - gnupg.Verify: The result of the verification + VerifyResult: The verification result with `valid`, `fingerprint`, `pubkey_fingerprint`, + `key_id`, `username`, and `data` attributes, mimicking the gnupg.Verify interface. Raises: pulpcore.exceptions.validation.InvalidSignatureError: In case the signature is invalid. """ - with tempfile.TemporaryDirectory(dir=settings.WORKING_DIRECTORY) as temp_directory_name: - gpg = gnupg.GPG(gnupghome=temp_directory_name) - gpg.import_keys(public_keys) - - with ExitStack() as stack: - if isinstance(signature, str): - signature = stack.enter_context(open(signature, "rb")) - elif isinstance(signature, models.Artifact): - signature = stack.enter_context(signature.file) - - verified = gpg.verify_file(signature, detached_data) - if not verified.valid: - raise InvalidSignatureError(_("The signature is not valid."), verified=verified) - return verified + from pysequoia import Cert, Sig, verify + + certs = Cert.split_bytes(public_keys.encode("utf8")) + + def store(key_ids): + return certs + + with ExitStack() as stack: + if isinstance(signature, str): + sig_data = stack.enter_context(open(signature, "rb")).read() + elif isinstance(signature, models.Artifact): + sig_data = stack.enter_context(signature.file).read() + else: + sig_data = signature.read() + + try: + sig = Sig.from_bytes(sig_data) + if detached_data is not None: + result = verify(file=detached_data, store=store, signature=sig) + else: + result = verify(bytes=sig_data, store=store) + except Exception: + raise InvalidSignatureError(_("The signature is not valid.")) + + return VerifyResult(result) def compute_file_hash(filename, hasher=None, cumulative_hash=None, blocksize=8192): diff --git a/pulpcore/exceptions/validation.py b/pulpcore/exceptions/validation.py index 67481723dbd..985aad65dec 100644 --- a/pulpcore/exceptions/validation.py +++ b/pulpcore/exceptions/validation.py @@ -109,7 +109,7 @@ def __str__(self): class InvalidSignatureError(ValidationError): """ - Raised when a signature could not be verified by the GnuPG utility. + Raised when a signature could not be verified. """ error_code = "PLP0021" diff --git a/pulpcore/pytest_plugin.py b/pulpcore/pytest_plugin.py index 35a886ffb0d..0a8de92a1ca 100644 --- a/pulpcore/pytest_plugin.py +++ b/pulpcore/pytest_plugin.py @@ -14,7 +14,6 @@ from time import sleep import aiohttp -import gnupg import pytest import requests from aiohttp import web @@ -1133,6 +1132,21 @@ def _sign_with_ascii_armored_detached_signing_service(filename): return _sign_with_ascii_armored_detached_signing_service +class _GpgCompat: + """Wrapper around a pysequoia Cert that provides the python-gnupg GPG interface needed by + downstream plugins (e.g. pulp_container) which access .gnupghome and .export_keys().""" + + def __init__(self, cert, gnupghome): + self.cert = cert + self.gnupghome = gnupghome + + def export_keys(self, keyids=None): + return str(self.cert) + + def __str__(self): + return str(self.cert) + + @pytest.fixture(scope="session") def signing_gpg_metadata(signing_gpg_homedir_path): """A fixture that returns a GPG instance and related metadata (i.e., fingerprint, keyid).""" @@ -1148,14 +1162,29 @@ def signing_gpg_metadata(signing_gpg_homedir_path): with suppress(FileNotFoundError, PermissionError): key_file.write_text(private_key_data) - gpg = gnupg.GPG(gnupghome=signing_gpg_homedir_path) - gpg.import_keys(private_key_data) + from pysequoia import Cert - fingerprint = gpg.list_keys()[0]["fingerprint"] - keyid = gpg.list_keys()[0]["keyid"] + cert = Cert.from_bytes(private_key_data.encode()) + fingerprint = cert.fingerprint.upper() + keyid = fingerprint[-16:] - gpg.trust_keys(fingerprint, "TRUST_ULTIMATE") + gpg_cmd = ["gpg", "--homedir", str(signing_gpg_homedir_path)] + subprocess.run( + gpg_cmd + ["--import"], + input=private_key_data, + capture_output=True, + text=True, + check=True, + ) + subprocess.run( + gpg_cmd + ["--import-ownertrust"], + input=f"{fingerprint}:6:\n", + capture_output=True, + text=True, + check=True, + ) + gpg = _GpgCompat(cert, str(signing_gpg_homedir_path)) return gpg, fingerprint, keyid @@ -1163,7 +1192,7 @@ def signing_gpg_metadata(signing_gpg_homedir_path): def pulp_trusted_public_key(signing_gpg_metadata): """Fixture to extract the ascii armored trusted public test key.""" gpg, _, keyid = signing_gpg_metadata - return gpg.export_keys([keyid]) + return str(gpg) @pytest.fixture(scope="session") @@ -1179,7 +1208,7 @@ def _ascii_armored_detached_signing_service_name( signing_gpg_homedir_path, ): service_name = str(uuid.uuid4()) - gpg, fingerprint, keyid = signing_gpg_metadata + _, fingerprint, keyid = signing_gpg_metadata cmd = ( "pulpcore-manager", diff --git a/pyproject.toml b/pyproject.toml index ed0d81c9498..da2a011f003 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -57,8 +57,7 @@ dependencies = [ "pygtrie>=2.5,<=2.5.0", "psycopg[binary]>=3.1.8,<3.4", # SemVer, not explicitely stated, but mentioned on multiple changes. "pyparsing>=3.1.0,<3.4", # Looks like only bugfixes in z-Stream. - "python-gnupg>=0.5.0,<0.6", # Looks like only bugfixes in z-Stream [changelog only in git] - "pysequoia==0.1.32", + "pysequoia==0.1.32", # Doesn't currently use semver, author has promised to use semver going forwards "PyYAML>=5.1.1,<6.1", # Looks like only bugfixes in z-Stream. "redis>=4.3.0,<7.2", # Looks like only bugfixes in z-Stream. "tablib>=3.5.0,<4.0, !=3.6", # 3.6.0 breaks with import export. Not sure about semver. From 1c24f0fbce5a5930e438ad05ddc2810d37ba3ecc Mon Sep 17 00:00:00 2001 From: Daniel Alley Date: Thu, 16 Apr 2026 18:16:23 -0400 Subject: [PATCH 03/10] Fix issues introduced by removing python-gnupg Restore python-gnupg to functional tests only - not worth trying to replace this fixture like-for-like Fix an issue with inlined signatures. We should only parse the sig separately in the detached case. (cherry picked from commit ac64ff3c7169b19610efde65cd72b129e04867e4) --- functest_requirements.txt | 2 +- pulpcore/app/util.py | 2 +- pulpcore/pytest_plugin.py | 47 ++++++++------------------------------- 3 files changed, 11 insertions(+), 40 deletions(-) diff --git a/functest_requirements.txt b/functest_requirements.txt index dce5f79b0db..c8bb7e387ce 100644 --- a/functest_requirements.txt +++ b/functest_requirements.txt @@ -1,7 +1,7 @@ pytest<10 pytest-custom_exit_code pytest-xdist -pysequoia +python-gnupg proxy.py~=2.4.10 trustme~=1.2.1 diff --git a/pulpcore/app/util.py b/pulpcore/app/util.py index 2b0a51f2f82..11c6698fbc4 100644 --- a/pulpcore/app/util.py +++ b/pulpcore/app/util.py @@ -441,8 +441,8 @@ def store(key_ids): sig_data = signature.read() try: - sig = Sig.from_bytes(sig_data) if detached_data is not None: + sig = Sig.from_bytes(sig_data) result = verify(file=detached_data, store=store, signature=sig) else: result = verify(bytes=sig_data, store=store) diff --git a/pulpcore/pytest_plugin.py b/pulpcore/pytest_plugin.py index 0a8de92a1ca..6ece66de735 100644 --- a/pulpcore/pytest_plugin.py +++ b/pulpcore/pytest_plugin.py @@ -1,4 +1,5 @@ import asyncio +import gnupg import json import os import pathlib @@ -1132,21 +1133,6 @@ def _sign_with_ascii_armored_detached_signing_service(filename): return _sign_with_ascii_armored_detached_signing_service -class _GpgCompat: - """Wrapper around a pysequoia Cert that provides the python-gnupg GPG interface needed by - downstream plugins (e.g. pulp_container) which access .gnupghome and .export_keys().""" - - def __init__(self, cert, gnupghome): - self.cert = cert - self.gnupghome = gnupghome - - def export_keys(self, keyids=None): - return str(self.cert) - - def __str__(self): - return str(self.cert) - - @pytest.fixture(scope="session") def signing_gpg_metadata(signing_gpg_homedir_path): """A fixture that returns a GPG instance and related metadata (i.e., fingerprint, keyid).""" @@ -1162,29 +1148,14 @@ def signing_gpg_metadata(signing_gpg_homedir_path): with suppress(FileNotFoundError, PermissionError): key_file.write_text(private_key_data) - from pysequoia import Cert + gpg = gnupg.GPG(gnupghome=signing_gpg_homedir_path) + gpg.import_keys(private_key_data) - cert = Cert.from_bytes(private_key_data.encode()) - fingerprint = cert.fingerprint.upper() - keyid = fingerprint[-16:] - - gpg_cmd = ["gpg", "--homedir", str(signing_gpg_homedir_path)] - subprocess.run( - gpg_cmd + ["--import"], - input=private_key_data, - capture_output=True, - text=True, - check=True, - ) - subprocess.run( - gpg_cmd + ["--import-ownertrust"], - input=f"{fingerprint}:6:\n", - capture_output=True, - text=True, - check=True, - ) + key = gpg.list_keys()[0] + fingerprint = key["fingerprint"] + keyid = key["keyid"] - gpg = _GpgCompat(cert, str(signing_gpg_homedir_path)) + gpg.trust_keys(fingerprint, "TRUST_ULTIMATE") return gpg, fingerprint, keyid @@ -1192,7 +1163,7 @@ def signing_gpg_metadata(signing_gpg_homedir_path): def pulp_trusted_public_key(signing_gpg_metadata): """Fixture to extract the ascii armored trusted public test key.""" gpg, _, keyid = signing_gpg_metadata - return str(gpg) + return gpg.export_keys([keyid]) @pytest.fixture(scope="session") @@ -1208,7 +1179,7 @@ def _ascii_armored_detached_signing_service_name( signing_gpg_homedir_path, ): service_name = str(uuid.uuid4()) - _, fingerprint, keyid = signing_gpg_metadata + _gpg, fingerprint, _keyid = signing_gpg_metadata cmd = ( "pulpcore-manager", From 53b445557cf9e8dcb464fae6dc6060bc50d126b4 Mon Sep 17 00:00:00 2001 From: Daniel Alley Date: Fri, 24 Apr 2026 09:36:03 -0400 Subject: [PATCH 04/10] Update pysequoia to fix gpg_verify() issues with some signatures (cherry picked from commit 1d46c31845284bb36d1aa070b942ae87d39af8cf) --- CHANGES/+gpg_verify.bugfix | 1 + pyproject.toml | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) create mode 100644 CHANGES/+gpg_verify.bugfix diff --git a/CHANGES/+gpg_verify.bugfix b/CHANGES/+gpg_verify.bugfix new file mode 100644 index 00000000000..06433d4faeb --- /dev/null +++ b/CHANGES/+gpg_verify.bugfix @@ -0,0 +1 @@ +Fixed an issue where gpg_verify() was rejecting some valid PGP signatures after changes made in pulpcore 3.108. diff --git a/pyproject.toml b/pyproject.toml index da2a011f003..d4836d38e82 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -57,7 +57,7 @@ dependencies = [ "pygtrie>=2.5,<=2.5.0", "psycopg[binary]>=3.1.8,<3.4", # SemVer, not explicitely stated, but mentioned on multiple changes. "pyparsing>=3.1.0,<3.4", # Looks like only bugfixes in z-Stream. - "pysequoia==0.1.32", # Doesn't currently use semver, author has promised to use semver going forwards + "pysequoia>=0.1.33,<0.2", "PyYAML>=5.1.1,<6.1", # Looks like only bugfixes in z-Stream. "redis>=4.3.0,<7.2", # Looks like only bugfixes in z-Stream. "tablib>=3.5.0,<4.0, !=3.6", # 3.6.0 breaks with import export. Not sure about semver. From a4fc2a091d68f41efcc62b83122ee41a0fad5f51 Mon Sep 17 00:00:00 2001 From: Daniel Alley Date: Wed, 12 Aug 2026 00:45:42 -0400 Subject: [PATCH 05/10] Bump pycryptography, pyOpenSSL versions We want the lower bound to have PQC support. (cherry picked from commit 2c01c5bc182ac81160ce19ba00beff60a1a27afe) --- pyproject.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index d4836d38e82..b6b75cb15f6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -31,7 +31,7 @@ dependencies = [ "asyncio-throttle>=1.0,<=1.0.2", # Unsure about versioning, but not released often anyway. "backoff>=2.1.2,<2.3", # Looks like only bugfixes in z-Stream. "click>=8.1.0,<8.4", # Uses milestone.feature.fix https://palletsprojects.com/versions . - "cryptography>=44.0.3,<47.0", # SemVer compatible https://cryptography.io/en/latest/api-stability/#versioning . + "cryptography>=49.0.0,<51.0", # SemVer compatible https://cryptography.io/en/latest/api-stability/#versioning . "Django>=4.2.24,<5.3, !=5.0, !=5.1", # LTS version, switch only if we have a compelling reason to". "django-filter>=23.1,<=25.2", # Uses CalVer, not released often https://github.com/carltongibson/django-filter "django-guid>=3.3.0,<3.6", # Looks like only bugfixes in z-Stream. @@ -48,7 +48,7 @@ dependencies = [ "jinja2>=3.1,<=3.1.6", "json_stream>=2.3.2,<2.5", "jq>=1.6.0,<1.12.0", - "PyOpenSSL<27.0", + "PyOpenSSL>=26.3.0,<27.0", "opentelemetry-api>=1.27.0,<1.41", "opentelemetry-sdk>=1.27.0,<1.41", "opentelemetry-exporter-otlp-proto-http>=1.27.0,<1.41", From 8c6576fa3acf486eb057de9f03f92fa3d2e70a18 Mon Sep 17 00:00:00 2001 From: Daniel Alley Date: Tue, 5 May 2026 23:41:43 -0400 Subject: [PATCH 06/10] Fix add-signing-service failing for keys with subkeys The previous implementation counted fpr: lines in GPG's colon output to verify that exactly one key matched the provided key ID. However, GPG emits a separate fpr: line for the primary key and each subkey, so any key with subkeys would be rejected with "There are N keys matching the key id." Count pub:/sec: lines instead, which represent actual distinct keys. Assisted-By: claude-opus-4.6 (cherry picked from commit 9d26e9522c32e623083ee89fd5bfb173042c807a) --- CHANGES/+fix-add-signing-service.bugfix | 1 + .../commands/add-signing-service.py | 15 +- pulpcore/pytest_plugin.py | 183 +++++++++++------- .../functional/api/test_signing_service.py | 23 +++ 4 files changed, 150 insertions(+), 72 deletions(-) create mode 100644 CHANGES/+fix-add-signing-service.bugfix diff --git a/CHANGES/+fix-add-signing-service.bugfix b/CHANGES/+fix-add-signing-service.bugfix new file mode 100644 index 00000000000..5e00ecf8b96 --- /dev/null +++ b/CHANGES/+fix-add-signing-service.bugfix @@ -0,0 +1 @@ +Fixed `add-signing-service` management command failing with "There are N keys matching the key id" for PGP keys that have subkeys. diff --git a/pulpcore/app/management/commands/add-signing-service.py b/pulpcore/app/management/commands/add-signing-service.py index 5c463da49c0..681ddbc1fd1 100644 --- a/pulpcore/app/management/commands/add-signing-service.py +++ b/pulpcore/app/management/commands/add-signing-service.py @@ -82,10 +82,17 @@ def handle(self, *args, **options): if result.returncode != 0: raise CommandError(result.stderr.strip()) - fpr_lines = [line for line in result.stdout.splitlines() if line.startswith("fpr:")] - if len(fpr_lines) != 1: - raise CommandError(_("There are {} keys matching the key id.").format(len(fpr_lines))) - fingerprint = fpr_lines[0].split(":")[9] + lines = result.stdout.splitlines() + + # Count actual keys (pub:/sec: lines), not fingerprint lines. GPG emits + # a separate fpr: line for the primary key and each subkey, so a single + # key with subkeys produces multiple fpr: lines. + key_lines = [l for l in lines if l.startswith(("pub:", "sec:"))] # noqa: E741 + if len(key_lines) != 1: + raise CommandError(_("There are {} keys matching the key id.").format(len(key_lines))) + + # Use the primary key fingerprint (first fpr: line in GPG's output). + fingerprint = [l.split(":")[9] for l in lines if l.startswith("fpr:")][0] # noqa: E741 result = subprocess.run( gpg_cmd + ["--armor", "--export", key_id], diff --git a/pulpcore/pytest_plugin.py b/pulpcore/pytest_plugin.py index 6ece66de735..98e602e9705 100644 --- a/pulpcore/pytest_plugin.py +++ b/pulpcore/pytest_plugin.py @@ -2,7 +2,6 @@ import gnupg import json import os -import pathlib import shutil import socket import ssl @@ -1060,43 +1059,61 @@ def _dispatch_task_group(task_name, *args, **kwargs): # GPG related fixtures +PULP_FIXTURES_SIGNING_KEYS_URL = ( + "https://raw.githubusercontent.com/pulp/pulp-fixtures/master/common/signing_keys/" +) +KEY_V4_RSA2K_PUBLIC = PULP_FIXTURES_SIGNING_KEYS_URL + "pulp-testkey-v4-rsa2k.asc" +KEY_V4_RSA2K_PRIVATE = PULP_FIXTURES_SIGNING_KEYS_URL + "pulp-testkey-v4-rsa2k.secret" +KEY_V4_RSA4K_PUBLIC = PULP_FIXTURES_SIGNING_KEYS_URL + "pulp-testkey-v4-rsa4k.asc" +KEY_V4_RSA4K_PRIVATE = PULP_FIXTURES_SIGNING_KEYS_URL + "pulp-testkey-v4-rsa4k.secret" +KEY_V4_ED25519_PUBLIC = PULP_FIXTURES_SIGNING_KEYS_URL + "pulp-testkey-v4-ed25519.asc" +KEY_V4_ED25519_PRIVATE = PULP_FIXTURES_SIGNING_KEYS_URL + "pulp-testkey-v4-ed25519.secret" +KEY_V6_RSA4K_PUBLIC = PULP_FIXTURES_SIGNING_KEYS_URL + "pulp-testkey-v6-rsa4k.asc" +KEY_V6_RSA4K_PRIVATE = PULP_FIXTURES_SIGNING_KEYS_URL + "pulp-testkey-v6-rsa4k.secret" +KEY_V6_ED25519_PUBLIC = PULP_FIXTURES_SIGNING_KEYS_URL + "pulp-testkey-v6-ed25519.asc" +KEY_V6_ED25519_PRIVATE = PULP_FIXTURES_SIGNING_KEYS_URL + "pulp-testkey-v6-ed25519.secret" +KEY_V6_MLDSA65_ED25519_PUBLIC = ( + PULP_FIXTURES_SIGNING_KEYS_URL + "pulp-testkey-v6-mldsa65-ed25519.asc" +) +KEY_V6_MLDSA65_ED25519_PRIVATE = ( + PULP_FIXTURES_SIGNING_KEYS_URL + "pulp-testkey-v6-mldsa65-ed25519.secret" +) +KEY_V6_MLDSA87_ED448_PUBLIC = PULP_FIXTURES_SIGNING_KEYS_URL + "pulp-testkey-v6-mldsa87-ed448.asc" +KEY_V6_MLDSA87_ED448_PRIVATE = ( + PULP_FIXTURES_SIGNING_KEYS_URL + "pulp-testkey-v6-mldsa87-ed448.secret" +) + -SIGNING_SCRIPT_STRING = r"""#!/usr/bin/env bash +SIGNING_SCRIPT_STRING = """#!/usr/bin/env bash FILE_PATH=$1 SIGNATURE_PATH="$1.asc" -GPG_KEY_ID="pulp-fixture-signing-key" +GPG_KEY_ID="{gpg_key_id}" # Create a detached signature -gpg --quiet --batch --homedir HOMEDIRHERE --detach-sign --local-user "${GPG_KEY_ID}" \ - --armor --output ${SIGNATURE_PATH} ${FILE_PATH} +gpg --quiet --batch --homedir {gpg_home} --detach-sign --local-user "${{GPG_KEY_ID}}" \\ + --armor --output ${{SIGNATURE_PATH}} ${{FILE_PATH}} # Check the exit status STATUS=$? -if [[ ${STATUS} -eq 0 ]]; then - echo {\"file\": \"${FILE_PATH}\", \"signature\": \"${SIGNATURE_PATH}\"} +if [[ ${{STATUS}} -eq 0 ]]; then + echo '{{"file": "'${{FILE_PATH}}'", "signature": "'${{SIGNATURE_PATH}}'"}}' else - exit ${STATUS} + exit ${{STATUS}} fi """ @pytest.fixture(scope="session") -def signing_script_path(signing_script_temp_dir, signing_gpg_homedir_path): - signing_script_file = signing_script_temp_dir / "sign-metadata.sh" - signing_script_file.write_text( - SIGNING_SCRIPT_STRING.replace("HOMEDIRHERE", str(signing_gpg_homedir_path)) - ) - - signing_script_file.chmod(0o755) - - return signing_script_file +def signing_script_path(signing_script_temp_dir, signing_gpg_homedir_path, signing_gpg_metadata): + _gpg, fingerprint, _keyid = signing_gpg_metadata + return make_signing_script(signing_gpg_homedir_path, fingerprint, signing_script_temp_dir) @pytest.fixture(scope="session") def signing_script_temp_dir(tmp_path_factory): - return tmp_path_factory.mktemp("sigining_script_dir") + return tmp_path_factory.mktemp("signing_script_dir") @pytest.fixture(scope="session") @@ -1137,26 +1154,7 @@ def _sign_with_ascii_armored_detached_signing_service(filename): def signing_gpg_metadata(signing_gpg_homedir_path): """A fixture that returns a GPG instance and related metadata (i.e., fingerprint, keyid).""" PRIVATE_KEY_URL = "https://raw.githubusercontent.com/pulp/pulp-fixtures/master/common/GPG-PRIVATE-KEY-fixture-signing" # noqa: E501 - - key_file = pathlib.Path(__file__).parent / "GPG-PRIVATE-KEY-fixture-signing" - if key_file.exists(): - private_key_data = key_file.read_text() - else: - response = requests.get(PRIVATE_KEY_URL) - response.raise_for_status() - private_key_data = response.text - with suppress(FileNotFoundError, PermissionError): - key_file.write_text(private_key_data) - - gpg = gnupg.GPG(gnupghome=signing_gpg_homedir_path) - gpg.import_keys(private_key_data) - - key = gpg.list_keys()[0] - fingerprint = key["fingerprint"] - keyid = key["keyid"] - - gpg.trust_keys(fingerprint, "TRUST_ULTIMATE") - return gpg, fingerprint, keyid + return import_signing_key(PRIVATE_KEY_URL, signing_gpg_homedir_path) @pytest.fixture(scope="session") @@ -1178,53 +1176,102 @@ def _ascii_armored_detached_signing_service_name( signing_gpg_metadata, signing_gpg_homedir_path, ): - service_name = str(uuid.uuid4()) _gpg, fingerprint, _keyid = signing_gpg_metadata + service_name = create_signing_service( + signing_gpg_homedir_path, fingerprint, signing_script_path + ) + + yield service_name + + remove_signing_service(service_name) + + +@pytest.fixture(scope="session") +def ascii_armored_detached_signing_service( + _ascii_armored_detached_signing_service_name, pulpcore_bindings +): + return pulpcore_bindings.SigningServicesApi.list( + name=_ascii_armored_detached_signing_service_name + ).results[0] + +def import_signing_key(key_url, gpg_home): + """Import a PGP key into a GPG home directory and trust it. + + Returns ``(gpg, fingerprint, keyid)``. + """ + try: + import gnupg + except ImportError: + pytest.skip("python-gnupg not installed") + + gpg = gnupg.GPG(gnupghome=gpg_home) + + response = requests.get(key_url) + response.raise_for_status() + result = gpg.import_keys(response.content) + assert result.count >= 1, f"Failed to import key from {key_url}" + + key_info = gpg.list_keys()[0] + fingerprint = key_info["fingerprint"] + keyid = key_info["keyid"] + gpg.trust_keys(fingerprint, "TRUST_ULTIMATE") + + return gpg, fingerprint, keyid + + +def make_signing_script(gpg_home, fingerprint, script_dir=None): + """Create a detached-signature signing script. + + Returns the script path. + """ + if script_dir is None: + script_dir = gpg_home + script_path = script_dir / "sign.sh" + script_path.write_text(SIGNING_SCRIPT_STRING.format(gpg_home=gpg_home, gpg_key_id=fingerprint)) + script_path.chmod(0o755) + return script_path + + +def create_signing_service( + gpg_home, fingerprint, script_path, *, service_class="core:AsciiArmoredDetachedSigningService" +): + """Register a signing service via pulpcore-manager. + + Returns the service name. + """ + service_name = str(uuid.uuid4()) cmd = ( "pulpcore-manager", "add-signing-service", service_name, - str(signing_script_path), + str(script_path), fingerprint, "--class", - "core:AsciiArmoredDetachedSigningService", + service_class, "--gnupghome", - str(signing_gpg_homedir_path), - ) - completed_process = subprocess.run( - cmd, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, + str(gpg_home), ) + completed = subprocess.run(cmd, capture_output=True, text=True) + assert completed.returncode == 0, completed.stderr - assert completed_process.returncode == 0 + return service_name - yield service_name - cmd = ( - "pulpcore-manager", - "remove-signing-service", - service_name, - "--class", - "core:AsciiArmoredDetachedSigningService", - ) +def remove_signing_service(service_name, service_class="core:AsciiArmoredDetachedSigningService"): + """Remove a signing service created by ``create_signing_service``.""" subprocess.run( - cmd, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, + ( + "pulpcore-manager", + "remove-signing-service", + service_name, + "--class", + service_class, + ), + capture_output=True, ) -@pytest.fixture(scope="session") -def ascii_armored_detached_signing_service( - _ascii_armored_detached_signing_service_name, pulpcore_bindings -): - return pulpcore_bindings.SigningServicesApi.list( - name=_ascii_armored_detached_signing_service_name - ).results[0] - - # if content_origin == None, base_url will return the relative path and # we need to add the hostname to run the tests @pytest.fixture(scope="session") diff --git a/pulpcore/tests/functional/api/test_signing_service.py b/pulpcore/tests/functional/api/test_signing_service.py index 25f4a477d40..78d875a6252 100644 --- a/pulpcore/tests/functional/api/test_signing_service.py +++ b/pulpcore/tests/functional/api/test_signing_service.py @@ -1,7 +1,30 @@ import pytest +from pulpcore.pytest_plugin import ( + KEY_V4_RSA4K_PRIVATE, + create_signing_service, + import_signing_key, + make_signing_script, + remove_signing_service, +) + @pytest.mark.parallel def test_crud_signing_service(ascii_armored_detached_signing_service): service = ascii_armored_detached_signing_service assert "/api/v3/signing-services/" in service.pulp_href + + +def test_add_signing_service_key_with_subkeys(tmp_path_factory): + """Verify that add-signing-service works with a PGP key that has subkeys. + + Keys with signing subkeys produce multiple fpr: lines in GPG's colon + output, which previously caused add-signing-service to fail. + """ + gpg_home = tmp_path_factory.mktemp("gpghome_subkey_test") + _gpg, fingerprint, _keyid = import_signing_key(KEY_V4_RSA4K_PRIVATE, gpg_home) + script_path = make_signing_script(gpg_home, fingerprint) + service_name = create_signing_service(gpg_home, fingerprint, script_path) + assert len(fingerprint) == 40 + + remove_signing_service(service_name) From c24a5cef0890c495e89eaf8169f546693d6e81e1 Mon Sep 17 00:00:00 2001 From: Daniel Alley Date: Wed, 5 Aug 2026 17:54:24 -0400 Subject: [PATCH 07/10] Add Sequoia (sq) backend support to add-signing-service The add-signing-service management command previously hardcoded GPG for key metadata extraction, preventing use with key types GPG cannot handle (OpenPGP v6, ML-DSA/post-quantum). This adds a --backend option that accepts "gpg" (default, existing behavior) or "sq" (Sequoia). Both backends reuse --gnupghome and --keyring, mapped to the equivalent sq CLI flags. The sq backend uses `sq cert export` to retrieve the public key and pysequoia to parse the fingerprint. Test infrastructure gains parallel Sequoia fixtures and helpers (import_signing_key_sq, make_sq_signing_script, create_signing_service_sq) and both signing service tests are parametrized to run with both backends. Assisted-By: Claude Opus 4.6 closes #7479 (cherry picked from commit a66a551f62d36f5ae29d215a7998b8c1f4afc612) --- CHANGES/7479.feature | 1 + .../commands/add-signing-service.py | 123 +++++++++++--- pulpcore/pytest_plugin.py | 152 +++++++++++++++--- .../functional/api/test_signing_service.py | 29 +++- 4 files changed, 249 insertions(+), 56 deletions(-) create mode 100644 CHANGES/7479.feature diff --git a/CHANGES/7479.feature b/CHANGES/7479.feature new file mode 100644 index 00000000000..6dab6beeed6 --- /dev/null +++ b/CHANGES/7479.feature @@ -0,0 +1 @@ +Added `--backend` option to the `add-signing-service` management command, enabling Sequoia (`sq`) as an alternative to GPG for key management. Use `--backend sq` to register signing services using Sequoia's key store. diff --git a/pulpcore/app/management/commands/add-signing-service.py b/pulpcore/app/management/commands/add-signing-service.py index 681ddbc1fd1..44f74d9dc18 100644 --- a/pulpcore/app/management/commands/add-signing-service.py +++ b/pulpcore/app/management/commands/add-signing-service.py @@ -1,5 +1,6 @@ import os import subprocess +import warnings from gettext import gettext as _ from pathlib import Path @@ -9,6 +10,11 @@ from pulpcore.app.models.content import SigningService as BaseSigningService +ENV_DEFAULTS = { + "gpg": "GNUPGHOME", + "sq": "SEQUOIA_HOME", +} + class Command(BaseCommand): """ @@ -28,7 +34,7 @@ def add_arguments(self, parser): ) parser.add_argument( "key", - help=_("Key id of the public key."), + help=_("Key id or fingerprint of the public key."), ) parser.add_argument( "--class", @@ -36,11 +42,27 @@ def add_arguments(self, parser): required=False, help=_("Signing service class prefixed by the app label separated by a colon."), ) + parser.add_argument( + "--backend", + choices=["gpg", "sq"], + default="gpg", + required=False, + help=_("Key management backend to use for extracting key metadata. (default: gpg)"), + ) + parser.add_argument( + "--home", + default=None, + required=False, + help=_( + "Home directory for the key management backend. " + "Defaults to $GNUPGHOME (gpg) or $SEQUOIA_HOME (sq)." + ), + ) parser.add_argument( "--gnupghome", - default=os.getenv("GNUPGHOME", ""), + default=None, required=False, - help=_("A default GnuPG home directory to use during the initialization."), + help=_("Deprecated: use --home instead."), ) parser.add_argument( "--keyring", @@ -68,11 +90,56 @@ def handle(self, *args, **options): ) ) - gpg_cmd = ["gpg"] + backend = options["backend"] + + if options["home"] and options["gnupghome"]: + raise CommandError(_("--home and --gnupghome are mutually exclusive.")) + if options["gnupghome"]: - gpg_cmd += ["--homedir", options["gnupghome"]] - if options["keyring"]: - gpg_cmd += ["--keyring", options["keyring"]] + warnings.warn( + "--gnupghome is deprecated; use --home instead.", + DeprecationWarning, + stacklevel=2, + ) + + home = options["home"] or options["gnupghome"] or os.getenv(ENV_DEFAULTS[backend], "") + + if backend == "sq": + fingerprint, public_key = self._extract_key_from_sq( + key_id, home, options.get("keyring") + ) + else: + fingerprint, public_key = self._extract_key_from_gpg( + key_id, home, options.get("keyring") + ) + + try: + script_path = Path(script).resolve(strict=True) + except FileNotFoundError as e: + raise CommandError(str(e)) + + try: + SigningService.objects.create( + name=name, + public_key=public_key, + pubkey_fingerprint=fingerprint, + script=script_path, + ) + except IntegrityError as e: + raise CommandError(str(e)) + + print( + ("Successfully added signing service {name} for key {fingerprint}.").format( + name=name, fingerprint=fingerprint + ) + ) + + def _extract_key_from_gpg(self, key_id, home, keyring): + gpg_cmd = ["gpg"] + if home: + gpg_cmd += ["--homedir", home] + if keyring: + gpg_cmd += ["--keyring", keyring] result = subprocess.run( gpg_cmd + ["--with-colons", "--fingerprint", key_id], @@ -103,23 +170,33 @@ def handle(self, *args, **options): raise CommandError(result.stderr.strip()) public_key = result.stdout - try: - script_path = Path(script).resolve(strict=True) - except FileNotFoundError as e: - raise CommandError(str(e)) + return fingerprint, public_key + + def _extract_key_from_sq(self, key_id, home, keyring): + from pysequoia import Cert + + sq_cmd = ["sq"] + if home: + sq_cmd += ["--home", home] + if keyring: + sq_cmd += ["--keyring", keyring] + + result = subprocess.run( + sq_cmd + ["cert", "export", "--cert", key_id], + capture_output=True, + text=True, + ) + if result.returncode != 0: + raise CommandError(result.stderr.strip()) + public_key = result.stdout try: - SigningService.objects.create( - name=name, - public_key=public_key, - pubkey_fingerprint=fingerprint, - script=script_path, + cert = Cert.from_bytes(public_key.encode("utf-8")) + except Exception as e: + raise CommandError( + _("Failed to parse exported certificate for '{}': {}").format(key_id, e) ) - except IntegrityError as e: - raise CommandError(str(e)) - print( - ("Successfully added signing service {name} for key {fingerprint}.").format( - name=name, fingerprint=fingerprint - ) - ) + fingerprint = cert.fingerprint.upper() + + return fingerprint, public_key diff --git a/pulpcore/pytest_plugin.py b/pulpcore/pytest_plugin.py index 98e602e9705..7be63fae3c4 100644 --- a/pulpcore/pytest_plugin.py +++ b/pulpcore/pytest_plugin.py @@ -1104,6 +1104,27 @@ def _dispatch_task_group(task_name, *args, **kwargs): fi """ +SQ_SIGNING_SCRIPT_STRING = """#!/usr/bin/env bash + +FILE_PATH=$1 +SIGNATURE_PATH="$1.asc" + +SQ_HOME="{sq_home}" +SIGNER="{signer_fingerprint}" + +# Create a detached signature using Sequoia (sq) +sq --home "${{SQ_HOME}}" sign --signer "${{SIGNER}}" \\ + --signature-file="${{SIGNATURE_PATH}}" "${{FILE_PATH}}" + +# Check the exit status +STATUS=$? +if [[ ${{STATUS}} -eq 0 ]]; then + echo '{{"file": "'${{FILE_PATH}}'", "signature": "'${{SIGNATURE_PATH}}'"}}' +else + exit ${{STATUS}} +fi +""" + @pytest.fixture(scope="session") def signing_script_path(signing_script_temp_dir, signing_gpg_homedir_path, signing_gpg_metadata): @@ -1195,53 +1216,132 @@ def ascii_armored_detached_signing_service( ).results[0] -def import_signing_key(key_url, gpg_home): - """Import a PGP key into a GPG home directory and trust it. +@pytest.fixture(scope="session") +def sq_signing_home_path(tmp_path_factory): + return tmp_path_factory.mktemp("sq_home") - Returns ``(gpg, fingerprint, keyid)``. - """ - try: - import gnupg - except ImportError: - pytest.skip("python-gnupg not installed") - gpg = gnupg.GPG(gnupghome=gpg_home) +@pytest.fixture(scope="session") +def sq_signing_script_temp_dir(tmp_path_factory): + return tmp_path_factory.mktemp("sq_signing_script_dir") + + +@pytest.fixture(scope="session") +def sq_signing_metadata(sq_signing_home_path): + """A fixture that returns Sequoia signing metadata (i.e., fingerprint, keyid).""" + return import_signing_key(KEY_V6_ED25519_PRIVATE, sq_signing_home_path, backend="sq") + +@pytest.fixture(scope="session") +def sq_signing_script_path(sq_signing_script_temp_dir, sq_signing_home_path, sq_signing_metadata): + _sq, fingerprint, _keyid = sq_signing_metadata + return make_signing_script( + sq_signing_home_path, fingerprint, sq_signing_script_temp_dir, backend="sq" + ) + + +@pytest.fixture(scope="session") +def _sq_ascii_armored_detached_signing_service_name( + sq_signing_script_path, + sq_signing_metadata, + sq_signing_home_path, +): + _sq, fingerprint, _keyid = sq_signing_metadata + service_name = create_signing_service( + sq_signing_home_path, fingerprint, sq_signing_script_path, backend="sq" + ) + + yield service_name + + remove_signing_service(service_name) + + +@pytest.fixture(scope="session") +def sq_ascii_armored_detached_signing_service( + _sq_ascii_armored_detached_signing_service_name, pulpcore_bindings +): + return pulpcore_bindings.SigningServicesApi.list( + name=_sq_ascii_armored_detached_signing_service_name + ).results[0] + + +def import_signing_key(key_url, home, *, backend="gpg"): + """Import a PGP key into a keyring and return metadata. + + Returns `(gpg_instance_or_none, fingerprint, keyid)`. The first element + is a `gnupg.GPG` instance when `backend` is `"gpg"`, or `None` when + `backend` is `"sq"`. + """ response = requests.get(key_url) response.raise_for_status() - result = gpg.import_keys(response.content) - assert result.count >= 1, f"Failed to import key from {key_url}" - key_info = gpg.list_keys()[0] - fingerprint = key_info["fingerprint"] - keyid = key_info["keyid"] - gpg.trust_keys(fingerprint, "TRUST_ULTIMATE") + if backend == "sq": + from pysequoia import Cert + + completed = subprocess.run( + ("sq", "--home", str(home), "key", "import"), + input=response.content, + capture_output=True, + ) + assert completed.returncode == 0, completed.stderr.decode() + + cert = Cert.from_bytes(response.content) + fingerprint = cert.fingerprint.upper() + keyid = fingerprint[-16:] + + return None, fingerprint, keyid + else: + try: + import gnupg + except ImportError: + pytest.skip("python-gnupg not installed") + + gpg = gnupg.GPG(gnupghome=home) + + result = gpg.import_keys(response.content) + assert result.count >= 1, f"Failed to import key from {key_url}" - return gpg, fingerprint, keyid + key_info = gpg.list_keys()[0] + fingerprint = key_info["fingerprint"] + keyid = key_info["keyid"] + gpg.trust_keys(fingerprint, "TRUST_ULTIMATE") + return gpg, fingerprint, keyid -def make_signing_script(gpg_home, fingerprint, script_dir=None): + +def make_signing_script(home, fingerprint, script_dir=None, *, backend="gpg"): """Create a detached-signature signing script. Returns the script path. """ if script_dir is None: - script_dir = gpg_home - script_path = script_dir / "sign.sh" - script_path.write_text(SIGNING_SCRIPT_STRING.format(gpg_home=gpg_home, gpg_key_id=fingerprint)) + script_dir = home + if backend == "sq": + script_path = script_dir / "sq_sign.sh" + script_path.write_text( + SQ_SIGNING_SCRIPT_STRING.format(sq_home=home, signer_fingerprint=fingerprint) + ) + else: + script_path = script_dir / "sign.sh" + script_path.write_text(SIGNING_SCRIPT_STRING.format(gpg_home=home, gpg_key_id=fingerprint)) script_path.chmod(0o755) return script_path def create_signing_service( - gpg_home, fingerprint, script_path, *, service_class="core:AsciiArmoredDetachedSigningService" + home, + fingerprint, + script_path, + *, + backend="gpg", + service_class="core:AsciiArmoredDetachedSigningService", ): """Register a signing service via pulpcore-manager. Returns the service name. """ service_name = str(uuid.uuid4()) - cmd = ( + cmd = [ "pulpcore-manager", "add-signing-service", service_name, @@ -1249,9 +1349,11 @@ def create_signing_service( fingerprint, "--class", service_class, - "--gnupghome", - str(gpg_home), - ) + "--backend", + backend, + "--home", + str(home), + ] completed = subprocess.run(cmd, capture_output=True, text=True) assert completed.returncode == 0, completed.stderr diff --git a/pulpcore/tests/functional/api/test_signing_service.py b/pulpcore/tests/functional/api/test_signing_service.py index 78d875a6252..717b32fe63b 100644 --- a/pulpcore/tests/functional/api/test_signing_service.py +++ b/pulpcore/tests/functional/api/test_signing_service.py @@ -10,21 +10,34 @@ @pytest.mark.parallel -def test_crud_signing_service(ascii_armored_detached_signing_service): - service = ascii_armored_detached_signing_service +@pytest.mark.parametrize( + "signing_service_fixture", + [ + "ascii_armored_detached_signing_service", + "sq_ascii_armored_detached_signing_service", + ], +) +def test_crud_signing_service(signing_service_fixture, request): + service = request.getfixturevalue(signing_service_fixture) assert "/api/v3/signing-services/" in service.pulp_href -def test_add_signing_service_key_with_subkeys(tmp_path_factory): +@pytest.mark.parametrize("backend", ["gpg", "sq"]) +def test_add_signing_service_key_with_subkeys(backend, tmp_path_factory): """Verify that add-signing-service works with a PGP key that has subkeys. Keys with signing subkeys produce multiple fpr: lines in GPG's colon output, which previously caused add-signing-service to fail. + + With both GPG and Sequoia backends, the service should be created + successfully with the primary key fingerprint. """ - gpg_home = tmp_path_factory.mktemp("gpghome_subkey_test") - _gpg, fingerprint, _keyid = import_signing_key(KEY_V4_RSA4K_PRIVATE, gpg_home) - script_path = make_signing_script(gpg_home, fingerprint) - service_name = create_signing_service(gpg_home, fingerprint, script_path) - assert len(fingerprint) == 40 + home = tmp_path_factory.mktemp(f"{backend}_subkey_test") + script_dir = tmp_path_factory.mktemp(f"{backend}_subkey_script") + _gpg, fingerprint, _keyid = import_signing_key(KEY_V4_RSA4K_PRIVATE, home, backend=backend) + script_path = make_signing_script(home, fingerprint, script_dir, backend=backend) + service_name = create_signing_service(home, fingerprint, script_path, backend=backend) + + assert len(fingerprint) in (40, 64) remove_signing_service(service_name) From 8801e83092dce3e10ddc5ba77ea5f719c4ffd434 Mon Sep 17 00:00:00 2001 From: Daniel Alley Date: Tue, 11 Aug 2026 15:26:30 -0400 Subject: [PATCH 08/10] Add PQC (ML-DSA) certificate test coverage Test scenarios added: Certguard (parameterized over ML-DSA-65 and ML-DSA-87): - Download with a valid PQC client cert returns 200 - Download with an untrusted PQC client cert returns 403 - Download with no client cert returns 403 Remote sync (ML-DSA-65): - on_demand sync over HTTPS with PQC server certificate (TLS validation) - on_demand sync over HTTPS with PQC mutual TLS (client cert required) Static PQC cert/key assets and a generation script are included for both ML-DSA-65 and ML-DSA-87. Certs are generated using pycryptography's x509 builder API. Assisted-By: Claude Opus 4.6 (cherry picked from commit 03c3d76a34cce9d73e47fa67c2c83f571957b768) --- functest_requirements.txt | 1 + .../functional/api/test_x509_certguard.py | 78 +++++++ pulp_certguard/tests/functional/constants.py | 1 - pulp_file/pytest_plugin.py | 58 ++++++ .../functional/api/test_remote_settings.py | 46 +++++ pulpcore/pytest_plugin.py | 195 +++++++++++++++++- template_config.yml | 3 + 7 files changed, 380 insertions(+), 2 deletions(-) diff --git a/functest_requirements.txt b/functest_requirements.txt index c8bb7e387ce..1a919656eab 100644 --- a/functest_requirements.txt +++ b/functest_requirements.txt @@ -4,6 +4,7 @@ pytest-xdist python-gnupg proxy.py~=2.4.10 trustme~=1.2.1 +cryptography>=49.0 # pulp_file tests beautifulsoup4 diff --git a/pulp_certguard/tests/functional/api/test_x509_certguard.py b/pulp_certguard/tests/functional/api/test_x509_certguard.py index bff81b03f67..3fad8a75109 100644 --- a/pulp_certguard/tests/functional/api/test_x509_certguard.py +++ b/pulp_certguard/tests/functional/api/test_x509_certguard.py @@ -1,4 +1,5 @@ import uuid +from collections import namedtuple from urllib.parse import quote, urljoin import pytest @@ -12,6 +13,11 @@ X509_UNTRUSTED_CLIENT_CERT_FILE_PATH, ) +PQCGuardedDistribution = namedtuple( + "PQCGuardedDistribution", + ["distribution", "algorithm", "client_cert_pem", "untrusted_client_cert_pem"], +) + @pytest.fixture(scope="class") def x509_certguard_factory(x509_content_guards_api_client, gen_object_with_cleanup): @@ -42,6 +48,31 @@ def x509_guarded_distribution( return distribution +@pytest.fixture(scope="class") +def pqc_guarded_distribution( + pqc_certificate_authority, + x509_content_guards_api_client, + gen_object_with_cleanup, + file_distribution_factory, + repository_test_file, +): + pca = pqc_certificate_authority + content_guard = gen_object_with_cleanup( + x509_content_guards_api_client, + { + "name": str(uuid.uuid4()), + "ca_certificate": pca.ca_cert_pem, + }, + ) + distribution = file_distribution_factory( + repository=repository_test_file.pulp_href, + content_guard=content_guard.pulp_href, + ) + return PQCGuardedDistribution( + distribution, pca.algorithm, pca.client_cert_pem, pca.untrusted_client_cert_pem + ) + + @pytest.fixture( scope="module", params=[ @@ -92,3 +123,50 @@ def test_download( headers=cert_data and {"X-CLIENT-CERT": cert_data}, ) assert response.status_code == status_code + + +class TestPQCX509CertGuard: + """Test X.509 content guard with PQC (ML-DSA) certificates. + + Parameterized over ML-DSA-65 (~7.5KB, under the default 8190-byte header + limit) and ML-DSA-87 (~10KB, over the limit). + """ + + def test_download_with_valid_cert( + self, + pqc_guarded_distribution, + distribution_base_url, + ): + distribution = pqc_guarded_distribution.distribution + url = distribution_base_url(distribution.base_url) + cert_pem = quote(pqc_guarded_distribution.client_cert_pem) + response = requests.get( + urljoin(url, "test_file"), + headers={"X-CLIENT-CERT": cert_pem}, + allow_redirects=False, + ) + assert response.status_code in (200, 302) + + def test_download_with_untrusted_cert( + self, + pqc_guarded_distribution, + distribution_base_url, + ): + distribution = pqc_guarded_distribution.distribution + url = distribution_base_url(distribution.base_url) + cert_pem = quote(pqc_guarded_distribution.untrusted_client_cert_pem) + response = requests.get( + urljoin(url, "test_file"), + headers={"X-CLIENT-CERT": cert_pem}, + ) + assert response.status_code == 403 + + def test_download_with_no_cert( + self, + pqc_guarded_distribution, + distribution_base_url, + ): + distribution = pqc_guarded_distribution.distribution + url = distribution_base_url(distribution.base_url) + response = requests.get(urljoin(url, "test_file")) + assert response.status_code == 403 diff --git a/pulp_certguard/tests/functional/constants.py b/pulp_certguard/tests/functional/constants.py index f481b1adb87..f0d04b2658a 100644 --- a/pulp_certguard/tests/functional/constants.py +++ b/pulp_certguard/tests/functional/constants.py @@ -14,7 +14,6 @@ X509_CERTS_BASE_PATH, "un_urlencoded_cert.txt" ) - RHSM_CA_CERT_FILE_PATH = os.path.join(_CURRENT_DIR, "artifacts", "rhsm", "katello-default-ca.crt") RHSM_CLIENT_CERT_FROM_UNTRUSTED_CA = os.path.join( diff --git a/pulp_file/pytest_plugin.py b/pulp_file/pytest_plugin.py index 62c83df846b..baa9f850830 100644 --- a/pulp_file/pytest_plugin.py +++ b/pulp_file/pytest_plugin.py @@ -255,6 +255,64 @@ def _file_remote_client_cert_req_factory(*, manifest_path, policy, **kwargs): return _file_remote_client_cert_req_factory +@pytest.fixture(scope="class") +def file_fixture_server_pqc_ssl(pqc_ssl_ctx, file_fixtures_root, gen_fixture_server): + return gen_fixture_server(file_fixtures_root, pqc_ssl_ctx) + + +@pytest.fixture(scope="class") +def file_fixture_server_pqc_ssl_client_cert_req( + pqc_ssl_ctx_req_client_auth, file_fixtures_root, gen_fixture_server +): + return gen_fixture_server(file_fixtures_root, pqc_ssl_ctx_req_client_auth) + + +@pytest.fixture(scope="class") +def file_remote_pqc_ssl_factory( + file_fixture_server_pqc_ssl, + file_bindings, + pqc_certificate_authority, + gen_object_with_cleanup, +): + def _file_remote_pqc_ssl_factory(*, manifest_path, policy, **kwargs): + url = file_fixture_server_pqc_ssl.make_url(manifest_path) + kwargs.update( + { + "url": str(url), + "policy": policy, + "name": str(uuid.uuid4()), + "ca_cert": pqc_certificate_authority.ca_cert_pem, + } + ) + return gen_object_with_cleanup(file_bindings.RemotesFileApi, kwargs) + + return _file_remote_pqc_ssl_factory + + +@pytest.fixture(scope="class") +def file_remote_pqc_client_cert_req_factory( + file_fixture_server_pqc_ssl_client_cert_req, + file_bindings, + pqc_certificate_authority, + gen_object_with_cleanup, +): + def _file_remote_pqc_client_cert_req_factory(*, manifest_path, policy, **kwargs): + url = file_fixture_server_pqc_ssl_client_cert_req.make_url(manifest_path) + kwargs.update( + { + "url": str(url), + "policy": policy, + "name": str(uuid.uuid4()), + "ca_cert": pqc_certificate_authority.ca_cert_pem, + "client_cert": pqc_certificate_authority.client_cert_pem, + "client_key": pqc_certificate_authority.client_key_pem, + } + ) + return gen_object_with_cleanup(file_bindings.RemotesFileApi, kwargs) + + return _file_remote_pqc_client_cert_req_factory + + @pytest.fixture(scope="class") def file_repository_factory(file_bindings, gen_object_with_cleanup): """A factory to generate a File Repository with auto-deletion after the test run.""" diff --git a/pulp_file/tests/functional/api/test_remote_settings.py b/pulp_file/tests/functional/api/test_remote_settings.py index d4d37ff2525..17a93395fae 100644 --- a/pulp_file/tests/functional/api/test_remote_settings.py +++ b/pulp_file/tests/functional/api/test_remote_settings.py @@ -217,6 +217,52 @@ def test_http_sync_ssl_with_client_cert_req( ) +@pytest.mark.parallel +def test_http_sync_pqc_ssl_tls_validation_on( + file_bindings, + file_remote_pqc_ssl_factory, + file_repo, + basic_manifest_path, + monitor_task, +): + """ + Test file on_demand sync with https:// using PQC (ML-DSA-65) server certificate. + """ + remote_on_demand = file_remote_pqc_ssl_factory( + manifest_path=basic_manifest_path, policy="on_demand", tls_validation=True + ) + + _run_basic_sync_and_assert( + file_bindings, + remote_on_demand, + file_repo, + monitor_task, + ) + + +@pytest.mark.parallel +def test_http_sync_pqc_ssl_with_client_cert_req( + file_bindings, + file_remote_pqc_client_cert_req_factory, + file_repo, + basic_manifest_path, + monitor_task, +): + """ + Test file on_demand sync with https:// using PQC (ML-DSA-65) mutual TLS authentication. + """ + remote_on_demand = file_remote_pqc_client_cert_req_factory( + manifest_path=basic_manifest_path, policy="on_demand" + ) + + _run_basic_sync_and_assert( + file_bindings, + remote_on_demand, + file_repo, + monitor_task, + ) + + @pytest.mark.parallel def test_ondemand_to_immediate_sync( file_bindings, diff --git a/pulpcore/pytest_plugin.py b/pulpcore/pytest_plugin.py index 7be63fae3c4..2300d992716 100644 --- a/pulpcore/pytest_plugin.py +++ b/pulpcore/pytest_plugin.py @@ -1,5 +1,4 @@ import asyncio -import gnupg import json import os import shutil @@ -463,6 +462,200 @@ def ssl_ctx_req_client_auth( return ssl_ctx +# PQC (Post-Quantum Cryptography) TLS Fixtures + +# TODO pretty much all of this can be deleted when "trustme" gets ML-DSA support + + +@dataclass +class PQCCertificateAuthority: + algorithm: str + ca_cert_pem: str + server_cert_path: str + server_key_path: str + client_cert_pem: str + client_key_pem: str + untrusted_client_cert_pem: str + + +_PQC_KEY_CLASSES = { + "ML-DSA-44": "MLDSA44PrivateKey", + "ML-DSA-65": "MLDSA65PrivateKey", + "ML-DSA-87": "MLDSA87PrivateKey", +} + + +def _generate_pqc_certs(td, host, algorithm="ML-DSA-65"): + """Generate a PQC CA and issue server + client certs signed by it.""" + import datetime + import ipaddress + + from cryptography import x509 + from cryptography.hazmat.primitives.asymmetric import mldsa + from cryptography.hazmat.primitives.serialization import Encoding, NoEncryption, PrivateFormat + from cryptography.x509.oid import ExtendedKeyUsageOID, NameOID + + key_cls = getattr(mldsa, _PQC_KEY_CLASSES[algorithm]) + now = datetime.datetime.now(datetime.timezone.utc) + not_before = now - datetime.timedelta(minutes=1) + not_after = now + datetime.timedelta(days=1) + + ca_key = key_cls.generate() + ca_name = x509.Name([x509.NameAttribute(NameOID.COMMON_NAME, "PQC Test CA")]) + ca_cert = ( + x509.CertificateBuilder() + .subject_name(ca_name) + .issuer_name(ca_name) + .public_key(ca_key.public_key()) + .serial_number(x509.random_serial_number()) + .not_valid_before(not_before) + .not_valid_after(not_after) + .add_extension(x509.BasicConstraints(ca=True, path_length=None), critical=True) + .add_extension( + x509.KeyUsage( + digital_signature=True, + content_commitment=False, + key_encipherment=False, + data_encipherment=False, + key_agreement=False, + key_cert_sign=True, + crl_sign=True, + encipher_only=False, + decipher_only=False, + ), + critical=True, + ) + .sign(private_key=ca_key, algorithm=None) + ) + + san = x509.SubjectAlternativeName( + [x509.DNSName("localhost"), x509.IPAddress(ipaddress.ip_address(host))] + ) + + def _issue_cert(name, cn, eku_oid): + key = key_cls.generate() + cert = ( + x509.CertificateBuilder() + .subject_name(x509.Name([x509.NameAttribute(NameOID.COMMON_NAME, cn)])) + .issuer_name(ca_name) + .public_key(key.public_key()) + .serial_number(x509.random_serial_number()) + .not_valid_before(not_before) + .not_valid_after(not_after) + .add_extension(x509.BasicConstraints(ca=False, path_length=None), critical=True) + .add_extension( + x509.KeyUsage( + digital_signature=True, + content_commitment=False, + key_encipherment=False, + data_encipherment=False, + key_agreement=False, + key_cert_sign=False, + crl_sign=False, + encipher_only=False, + decipher_only=False, + ), + critical=True, + ) + .add_extension(x509.ExtendedKeyUsage([eku_oid]), critical=False) + .add_extension(san, critical=False) + .sign(private_key=ca_key, algorithm=None) + ) + cert_pem = cert.public_bytes(Encoding.PEM).decode() + key_pem = key.private_bytes(Encoding.PEM, PrivateFormat.PKCS8, NoEncryption()).decode() + cert_path = str(td / f"{name}.crt") + key_path = str(td / f"{name}.key") + with open(cert_path, "w") as f: + f.write(cert_pem) + with open(key_path, "w") as f: + f.write(key_pem) + return cert_path, key_path, cert_pem, key_pem + + ca_cert_pem = ca_cert.public_bytes(Encoding.PEM).decode() + + server_cert_path, server_key_path, _, _ = _issue_cert( + "server", host, ExtendedKeyUsageOID.SERVER_AUTH + ) + _, _, client_cert_pem, client_key_pem = _issue_cert( + "client", host, ExtendedKeyUsageOID.CLIENT_AUTH + ) + + # Untrusted client: signed by a different CA so cert-chain verification rejects it + untrusted_ca_key = key_cls.generate() + untrusted_ca_name = x509.Name([x509.NameAttribute(NameOID.COMMON_NAME, "PQC Untrusted CA")]) + untrusted_key = key_cls.generate() + untrusted_cert = ( + x509.CertificateBuilder() + .subject_name(x509.Name([x509.NameAttribute(NameOID.COMMON_NAME, "PQC Untrusted Client")])) + .issuer_name(untrusted_ca_name) + .public_key(untrusted_key.public_key()) + .serial_number(x509.random_serial_number()) + .not_valid_before(not_before) + .not_valid_after(not_after) + .add_extension(x509.BasicConstraints(ca=False, path_length=None), critical=True) + .add_extension( + x509.KeyUsage( + digital_signature=True, + content_commitment=False, + key_encipherment=False, + data_encipherment=False, + key_agreement=False, + key_cert_sign=False, + crl_sign=False, + encipher_only=False, + decipher_only=False, + ), + critical=True, + ) + .sign(private_key=untrusted_ca_key, algorithm=None) + ) + untrusted_client_cert_pem = untrusted_cert.public_bytes(Encoding.PEM).decode() + + return PQCCertificateAuthority( + algorithm=algorithm, + ca_cert_pem=ca_cert_pem, + server_cert_path=server_cert_path, + server_key_path=server_key_path, + client_cert_pem=client_cert_pem, + client_key_pem=client_key_pem, + untrusted_client_cert_pem=untrusted_client_cert_pem, + ) + + +@pytest.fixture(scope="session", params=["ML-DSA-65", "ML-DSA-87"]) +def pqc_certificate_authority(request, tmp_path_factory, fixtures_cfg): + """Generate a PQC Certificate Authority and related certs for each ML-DSA variant.""" + algorithm = request.param + td = tmp_path_factory.mktemp(f"pqc_certs_{algorithm}") + return _generate_pqc_certs(td, fixtures_cfg.aiohttp_fixtures_origin, algorithm) + + +@pytest.fixture(scope="session") +def pqc_ssl_ctx(pqc_certificate_authority): + """Server SSL context using PQC certificates (no client auth required).""" + ssl_ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) + ssl_ctx.load_cert_chain( + pqc_certificate_authority.server_cert_path, + pqc_certificate_authority.server_key_path, + ) + return ssl_ctx + + +@pytest.fixture(scope="session") +def pqc_ssl_ctx_req_client_auth(pqc_certificate_authority): + """Server SSL context using PQC certificates, requiring PQC client auth.""" + ssl_ctx = ssl.create_default_context( + purpose=ssl.Purpose.CLIENT_AUTH, + cadata=pqc_certificate_authority.ca_cert_pem, + ) + ssl_ctx.verify_mode = ssl.CERT_REQUIRED + ssl_ctx.load_cert_chain( + pqc_certificate_authority.server_cert_path, + pqc_certificate_authority.server_key_path, + ) + return ssl_ctx + + # Object factories diff --git a/template_config.yml b/template_config.yml index 98619948b6b..0b0e6838ce4 100644 --- a/template_config.yml +++ b/template_config.yml @@ -6,6 +6,9 @@ # After editing this file please always reapply the plugin template before committing any changes. --- +# NOTE: the S3 runner has redis disabled, domains enabled, and hide_guarded_distributions = true +# "weird" test failures unique to that runner may be due to those differences. + check_commit_message: true check_manifest: true check_stray_pulpcore_imports: false From 7e859bcb91c9e59958224bdc3a93dafa10bc7f87 Mon Sep 17 00:00:00 2001 From: Daniel Alley Date: Tue, 11 Aug 2026 16:05:59 -0400 Subject: [PATCH 09/10] Increase content app max HTTP header field size for PQC certificates Post-quantum (ML-DSA) X.509 certificates are significantly larger than traditional RSA/ECDSA certificates. When a reverse proxy forwards a PQC client certificate via the X-CLIENT-CERT header, it can exceed aiohttp's default 8190-byte max_field_size, causing a 400 LineTooLong error. Increase the limit to 16KB. Assisted-By: Claude Opus 4.6 (cherry picked from commit ab9a801efe465679353726922886455bf60f8ba8) --- CHANGES/+header-too-large.bugfix | 1 + pulpcore/content/__init__.py | 10 ++++++++-- 2 files changed, 9 insertions(+), 2 deletions(-) create mode 100644 CHANGES/+header-too-large.bugfix diff --git a/CHANGES/+header-too-large.bugfix b/CHANGES/+header-too-large.bugfix new file mode 100644 index 00000000000..a5106eec27e --- /dev/null +++ b/CHANGES/+header-too-large.bugfix @@ -0,0 +1 @@ +Increased the content app's maximum HTTP header field size from 8190 to 16384 bytes to support PQC (post-quantum) X.509 certificates forwarded via the `X-CLIENT-CERT` header. diff --git a/pulpcore/content/__init__.py b/pulpcore/content/__init__.py index decf0957b0d..26d5200de45 100644 --- a/pulpcore/content/__init__.py +++ b/pulpcore/content/__init__.py @@ -29,12 +29,18 @@ log = logging.getLogger(__name__) +# PQC (post-quantum) X.509 certificates can exceed aiohttp's default 8190-byte header +# limit when forwarded via X-CLIENT-CERT by a reverse proxy. +_HANDLER_ARGS = {"max_field_size": 16 * 1024} + if settings.OTEL_ENABLED: from .instrumentation import instrumentation # noqa: E402 - app = web.Application(middlewares=[guid, authenticate, instrumentation()]) + app = web.Application( + middlewares=[guid, authenticate, instrumentation()], handler_args=_HANDLER_ARGS + ) else: - app = web.Application(middlewares=[guid, authenticate]) + app = web.Application(middlewares=[guid, authenticate], handler_args=_HANDLER_ARGS) if settings.UVLOOP_ENABLED: From 238ce1847543694c097b486196d89fd4bfcdad07 Mon Sep 17 00:00:00 2001 From: Daniel Alley Date: Thu, 4 Jun 2026 09:38:48 -0400 Subject: [PATCH 10/10] Use a CS10-based CI image (cherry picked from commit 2d773881df0432ed227560a69f8630865127aec9) --- .github/workflows/scripts/before_install.sh | 2 +- MANIFEST.in | 1 + Makefile | 20 +++++++++++++++++++ .../api/using_plugin/test_content_delivery.py | 2 +- pyproject.toml | 2 +- template_config.yml | 2 +- 6 files changed, 25 insertions(+), 4 deletions(-) create mode 100644 Makefile diff --git a/.github/workflows/scripts/before_install.sh b/.github/workflows/scripts/before_install.sh index 95a1ba503cb..27cd6ec193d 100755 --- a/.github/workflows/scripts/before_install.sh +++ b/.github/workflows/scripts/before_install.sh @@ -55,7 +55,7 @@ pulp_scheme: "https" image: name: "pulp" tag: "ci_build" - ci_base: "ghcr.io/pulp/pulp-ci-centos9:latest" + ci_base: "ghcr.io/pulp/pulp-ci-centos10:latest" source: "${COMPONENT_SOURCE}" ci_requirements: $(test -f ci_requirements.txt && echo -n true || echo -n false) upperbounds: $(test "${TEST}" = "pulp" && echo -n true || echo -n false) diff --git a/MANIFEST.in b/MANIFEST.in index 862f03cc35e..fdda5e340fe 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -17,4 +17,5 @@ include test_requirements.txt exclude releasing.md exclude AGENTS.md exclude CLAUDE.md +exclude Makefile recursive-exclude pulpcore/tasking/task_trigger_demonstration * diff --git a/Makefile b/Makefile new file mode 100644 index 00000000000..b4228e5ba22 --- /dev/null +++ b/Makefile @@ -0,0 +1,20 @@ +# WARNING: DO NOT EDIT! +# +# This file was generated by plugin_template, and is managed by it. Please use +# './plugin-template --ci pulpcore' to update this file. +# +# For more info visit https://github.com/pulp/plugin_template + +.PHONY: format +format: + ruff format + ruff check --select I --fix + +.PHONY: lint +lint: + yamllint -s -d '{extends: relaxed, rules: {line-length: disable}}' .github/workflows + bump-my-version bump --dry-run --allow-dirty release + ruff format --check --diff + ruff check + check-manifest + python .ci/scripts/check_requirements.py diff --git a/pulpcore/tests/functional/api/using_plugin/test_content_delivery.py b/pulpcore/tests/functional/api/using_plugin/test_content_delivery.py index 0961e4f5350..c398be01372 100644 --- a/pulpcore/tests/functional/api/using_plugin/test_content_delivery.py +++ b/pulpcore/tests/functional/api/using_plugin/test_content_delivery.py @@ -154,7 +154,7 @@ def test_remote_content_changed_with_on_demand( # THEN assert not output_file.exists() assert result.returncode == 18 - assert b"* Closing connection 0" in result.stderr + assert b"closing connection" in result.stderr.lower() assert b"curl: (18) transfer closed with outstanding read data remaining" in result.stderr # WHEN (second request) diff --git a/pyproject.toml b/pyproject.toml index b6b75cb15f6..847b378b609 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -55,7 +55,7 @@ dependencies = [ "protobuf>=4.21.1,<7.0", "pulp-glue>=0.30.0,<0.40", "pygtrie>=2.5,<=2.5.0", - "psycopg[binary]>=3.1.8,<3.4", # SemVer, not explicitely stated, but mentioned on multiple changes. + "psycopg[binary]>=3.3.4,<3.4", # SemVer, not explicitely stated, but mentioned on multiple changes. "pyparsing>=3.1.0,<3.4", # Looks like only bugfixes in z-Stream. "pysequoia>=0.1.33,<0.2", "PyYAML>=5.1.1,<6.1", # Looks like only bugfixes in z-Stream. diff --git a/template_config.yml b/template_config.yml index 0b0e6838ce4..507b2bdcc1b 100644 --- a/template_config.yml +++ b/template_config.yml @@ -12,7 +12,7 @@ check_commit_message: true check_manifest: true check_stray_pulpcore_imports: false -ci_base_image: "ghcr.io/pulp/pulp-ci-centos9" +ci_base_image: "ghcr.io/pulp/pulp-ci-centos10" ci_env: {} ci_trigger: "{pull_request: {branches: ['*']}}" cli_package: "pulp-cli"