WIP: feat(upgrade): in-place Debian 12 to 13 relay upgrades

.github/workflows/nightly.yml

@@ -0,0 +1,51 @@
+name: Nightly relay integration
+
+on:
+  schedule:
+    - cron: "17 2 * * *"
+  workflow_dispatch:
+  # TEMPORARY: smoke-test on feature branch
+  push:
+    branches: [ j4n/nightly-matrix ]
+
+jobs:
+  nightly:
+    uses: ./.github/workflows/lxc-test.yml
+    with:
+      # TEMPORARY: lxc-test defaults to installing cmlxc from main, take our
+      # branch version instead until it's merged
+      cmlxc_version: ${{ github.sha }}
+      cmlxc_commands: |
+        cmlxc init
+        cmlxc deploy-cmdeploy --source @main fulltest0
+        cmlxc deploy-cmdeploy --source @main fulltest1
+        cmlxc test-mini fulltest0
+        cmlxc test-cmdeploy fulltest0 fulltest1
+        cmlxc stop fulltest0 fulltest1
+        cmlxc destroy fulltest1
+        cmlxc deploy-cmdeploy --type ipv4 --source @main fulltest-ip0
+        cmlxc test-mini fulltest-ip0
+        cmlxc test-cmdeploy fulltest-ip0
+        cmlxc deploy-madmail fulltest-mad0
+        cmlxc test-mini fulltest-mad0
+        cmlxc test-madmail fulltest-mad0
+        cmlxc test-mini fulltest0 fulltest-mad0
+        cmlxc test-mini fulltest-mad0 fulltest0
+        cmlxc test-mini fulltest-ip0 fulltest-mad0
+        cmlxc test-mini fulltest-mad0 fulltest-ip0
+        cmlxc test-mini fulltest-ip0 fulltest0
+        cmlxc test-mini fulltest0 fulltest-ip0
+
+  # Simulate in-place Debian 12 -> 13 dist-upgrade, separate parallel job;
+  # use @main once relay#1002 merges.
+  upgrade:
+    uses: ./.github/workflows/lxc-test.yml
+    with:
+      cmlxc_version: ${{ github.sha }}
+      cmlxc_commands: |
+        cmlxc init
+        cmlxc deploy-cmdeploy --source @j4n/dovecot-multidist upgrade0
+        cmlxc dist-upgrade upgrade0
+        cmlxc deploy-cmdeploy --source @j4n/dovecot-multidist upgrade0
+        cmlxc stop upgrade0 && cmlxc start upgrade0
+        cmlxc test-cmdeploy upgrade0

README.md

@@ -94,6 +94,7 @@ builder (wipe-and-reclone).
     cmlxc status cm0 mad1       # show multiple containers
     cmlxc status --host         # show DNS/SSH setup instructions
     cmlxc start cm0             # restart a stopped relay
+    cmlxc dist-upgrade cm0      # in-place Debian upgrade and redeploy
     cmlxc stop cm0 cm1          # stop relays
     cmlxc destroy cm0           # stop + delete
     cmlxc destroy --all         # destroy relays, keep DNS/builder

src/cmlxc/cli.py

@@ -176,10 +176,41 @@ def start_cmd(args, out):
             return 1
         out.green(f"Starting container {ct.name!r} ...")
         ct.start()
+        ct.wait_ready()
     ix.write_ssh_config()
     out.green("LXC containers started.")
 
 
+# -------------------------------------------------------------------
+# upgrade
+# -------------------------------------------------------------------
+
+
+def upgrade_cmd_options(parser):
+    parser.add_argument(
+        "names",
+        nargs="+",
+        metavar="NAME",
+        help="One or more relay containers to upgrade.",
+    ).completer = _container_completer
+
+
+def upgrade_cmd(args, out):
+    """Upgrade relay containers from Debian 12 to Debian 13.
+
+    Refuse containers without a deploy state (dns, builder).
+    """
+    ix = Incus(out)
+    for name in args.names:
+        ct = ix.get_running_relay(name)
+        if not ct.get_deploy_state():
+            raise SetupError(
+                f"{name!r} has no deploy state; only deployed relays can be upgraded."
+            )
+        with out.section(f"upgrade: {ct.shortname}"):
+            ct.upgrade_debian()
+
+
 # -------------------------------------------------------------------
 # stop
 # -------------------------------------------------------------------
@@ -608,6 +639,7 @@ def _print_dns_forwarding_status(out, dns_ip, *, host=False):
     ("test-mini", test_mini_cmd, test_mini_cmd_options),
     ("status", status_cmd, status_cmd_options),
     ("start", start_cmd, start_cmd_options),
+    ("dist-upgrade", upgrade_cmd, upgrade_cmd_options),
     ("stop", stop_cmd, stop_cmd_options),
     ("destroy", destroy_cmd, destroy_cmd_options),
 ]

src/cmlxc/container.py

@@ -40,6 +40,57 @@
 DNS_NS = "ns.localchat"
 DNS_CONTAINER_NAME = "ns-localchat"
 
+# In-place Debian 12 -> 13 upgrade, run inside a relay container.
+# --force-conf*: DEBIAN_FRONTEND silences debconf but not dpkg conffile
+# prompts, and we hand-edit resolv.conf and unbound config.
+UPGRADE_SCRIPT = r"""
+#!/bin/bash
+set -eux
+export DEBIAN_FRONTEND=noninteractive
+APT="apt-get -y -o DPkg::Lock::Timeout=300 \
+-o Dpkg::Options::=--force-confold \
+-o Dpkg::Options::=--force-confdef"
+# --allow-releaseinfo-change is an update-only option, apt errors out if it
+# is passed to install or full-upgrade
+APT_UPDATE="$APT update --allow-releaseinfo-change"
+
+# Ensure we have the dpkg lock
+systemctl disable --now unattended-upgrades \
+    apt-daily.timer apt-daily-upgrade.timer || true
+$APT purge unattended-upgrades || true
+
+# Backup our resolv.conf
+systemctl disable --now systemd-resolved || true
+systemctl mask systemd-resolved || true
+cp /etc/resolv.conf /root/resolv.conf.pre-upgrade
+
+# Allow libc6 restarts
+echo 'libraries/restart-without-asking boolean true' | debconf-set-selections
+
+# Upgrade to latest bookworm
+$APT_UPDATE
+$APT install debian-archive-keyring
+$APT full-upgrade
+
+# Rewrite both sources.list and .sources
+shopt -s nullglob
+for f in /etc/apt/sources.list /etc/apt/sources.list.d/*.list \
+         /etc/apt/sources.list.d/*.sources; do
+    sed -i 's/bookworm/trixie/g' "$f"
+done
+grep -rq trixie /etc/apt/sources.list /etc/apt/sources.list.d/
+
+$APT_UPDATE
+$APT full-upgrade
+$APT --purge autoremove
+apt-get clean
+
+cp /root/resolv.conf.pre-upgrade /etc/resolv.conf
+
+. /etc/os-release
+test "$VERSION_ID" = "13"
+"""
+
 
 class DNSConfigurationError(Exception):
     """Raised on DNS reachability or response failure."""
@@ -425,6 +476,31 @@ def check():
                 f"  Warning: Services on ports {ports} not ready after {timeout}s"
             )
 
+    def upgrade_debian(self):
+        """In-place dist-upgrade this relay from Debian 12 to 13.
+
+        Our apt-pin freezes dovecot at its bookworm build while the trixie
+        time_t transition removes libssl3/libtirpc3 underneath it, so dovecot
+        survives as an unrunnable package.  Redeploy afterwards.
+        """
+        path = "/root/cmlxc-upgrade.sh"
+        self.push_file_content(path, UPGRADE_SCRIPT, mode="755")
+        ret = self.out.shell(f"incus exec {self.name} -- {path}")
+        if ret:
+            raise SetupError(f"Debian upgrade failed on {self.shortname} (exit {ret})")
+
+        self.out.print("Restarting container after upgrade ...")
+        restart = self.incus.run(["restart", self.name, "--timeout=120"], check=False)
+        if restart.returncode:
+            self.out.red("Graceful restart timed out; forcing")
+            self.stop(force=True)
+            self.start()
+        self.wait_ready()
+        if not self.verify_ssh(self.incus.ssh_config_path):
+            raise SetupError(f"{self.shortname}: no SSH after upgrade reboot")
+        release = self.bash('. /etc/os-release; echo "$PRETTY_NAME"')
+        self.out.green(f"{self.shortname} upgraded: {release}")
+
     def verify_ssh(self, ssh_config):
         cmd = ["ssh", "-F", str(ssh_config), "-o", "ConnectTimeout=60"]
         cmd += [f"root@{self.domain}", "hostname"]

tests/test_cli.py

@@ -4,6 +4,7 @@
 
 import pytest
 
+from cmlxc.cli import get_parser, upgrade_cmd
 from cmlxc.driver_base import SourceSpec, parse_source, validate_relay_name
 from cmlxc.driver_cmdeploy import get_ini_overrides
 
@@ -61,3 +62,9 @@ def test_ini_overrides_disable_ipv6():
     assert "disable_ipv6" not in get_ini_overrides("cm0.localchat")
     overrides = get_ini_overrides("cm0.localchat", disable_ipv6=True)
     assert overrides["disable_ipv6"] == "True"
+
+
+def test_upgrade_parses_multiple_names():
+    args = get_parser().parse_args(["dist-upgrade", "cm0", "cm1"])
+    assert args.func is upgrade_cmd
+    assert args.names == ["cm0", "cm1"]