diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml new file mode 100644 index 0000000..e5a8c3e --- /dev/null +++ b/.github/workflows/nightly.yml @@ -0,0 +1,51 @@ +name: Nightly relay integration + +on: + schedule: + - cron: "17 2 * * *" + workflow_dispatch: + # TEMPORARY: smoke-test on feature branch + push: + branches: [ j4n/nightly-matrix ] + +jobs: + nightly: + uses: ./.github/workflows/lxc-test.yml + with: + # TEMPORARY: lxc-test defaults to installing cmlxc from main, take our + # branch version instead until it's merged + cmlxc_version: ${{ github.sha }} + cmlxc_commands: | + cmlxc init + cmlxc deploy-cmdeploy --source @main fulltest0 + cmlxc deploy-cmdeploy --source @main fulltest1 + cmlxc test-mini fulltest0 + cmlxc test-cmdeploy fulltest0 fulltest1 + cmlxc stop fulltest0 fulltest1 + cmlxc destroy fulltest1 + cmlxc deploy-cmdeploy --type ipv4 --source @main fulltest-ip0 + cmlxc test-mini fulltest-ip0 + cmlxc test-cmdeploy fulltest-ip0 + cmlxc deploy-madmail fulltest-mad0 + cmlxc test-mini fulltest-mad0 + cmlxc test-madmail fulltest-mad0 + cmlxc test-mini fulltest0 fulltest-mad0 + cmlxc test-mini fulltest-mad0 fulltest0 + cmlxc test-mini fulltest-ip0 fulltest-mad0 + cmlxc test-mini fulltest-mad0 fulltest-ip0 + cmlxc test-mini fulltest-ip0 fulltest0 + cmlxc test-mini fulltest0 fulltest-ip0 + + # Simulate in-place Debian 12 -> 13 dist-upgrade, separate parallel job; + # use @main once relay#1002 merges. + upgrade: + uses: ./.github/workflows/lxc-test.yml + with: + cmlxc_version: ${{ github.sha }} + cmlxc_commands: | + cmlxc init + cmlxc deploy-cmdeploy --source @j4n/dovecot-multidist upgrade0 + cmlxc dist-upgrade upgrade0 + cmlxc deploy-cmdeploy --source @j4n/dovecot-multidist upgrade0 + cmlxc stop upgrade0 && cmlxc start upgrade0 + cmlxc test-cmdeploy upgrade0 diff --git a/README.md b/README.md index 5586000..5bc6589 100644 --- a/README.md +++ b/README.md @@ -94,6 +94,7 @@ builder (wipe-and-reclone). cmlxc status cm0 mad1 # show multiple containers cmlxc status --host # show DNS/SSH setup instructions cmlxc start cm0 # restart a stopped relay + cmlxc dist-upgrade cm0 # in-place Debian upgrade and redeploy cmlxc stop cm0 cm1 # stop relays cmlxc destroy cm0 # stop + delete cmlxc destroy --all # destroy relays, keep DNS/builder diff --git a/src/cmlxc/cli.py b/src/cmlxc/cli.py index 02dff4c..88c36c1 100644 --- a/src/cmlxc/cli.py +++ b/src/cmlxc/cli.py @@ -176,10 +176,41 @@ def start_cmd(args, out): return 1 out.green(f"Starting container {ct.name!r} ...") ct.start() + ct.wait_ready() ix.write_ssh_config() out.green("LXC containers started.") +# ------------------------------------------------------------------- +# upgrade +# ------------------------------------------------------------------- + + +def upgrade_cmd_options(parser): + parser.add_argument( + "names", + nargs="+", + metavar="NAME", + help="One or more relay containers to upgrade.", + ).completer = _container_completer + + +def upgrade_cmd(args, out): + """Upgrade relay containers from Debian 12 to Debian 13. + + Refuse containers without a deploy state (dns, builder). + """ + ix = Incus(out) + for name in args.names: + ct = ix.get_running_relay(name) + if not ct.get_deploy_state(): + raise SetupError( + f"{name!r} has no deploy state; only deployed relays can be upgraded." + ) + with out.section(f"upgrade: {ct.shortname}"): + ct.upgrade_debian() + + # ------------------------------------------------------------------- # stop # ------------------------------------------------------------------- @@ -608,6 +639,7 @@ def _print_dns_forwarding_status(out, dns_ip, *, host=False): ("test-mini", test_mini_cmd, test_mini_cmd_options), ("status", status_cmd, status_cmd_options), ("start", start_cmd, start_cmd_options), + ("dist-upgrade", upgrade_cmd, upgrade_cmd_options), ("stop", stop_cmd, stop_cmd_options), ("destroy", destroy_cmd, destroy_cmd_options), ] diff --git a/src/cmlxc/container.py b/src/cmlxc/container.py index 9301d18..c17b8f0 100644 --- a/src/cmlxc/container.py +++ b/src/cmlxc/container.py @@ -40,6 +40,57 @@ DNS_NS = "ns.localchat" DNS_CONTAINER_NAME = "ns-localchat" +# In-place Debian 12 -> 13 upgrade, run inside a relay container. +# --force-conf*: DEBIAN_FRONTEND silences debconf but not dpkg conffile +# prompts, and we hand-edit resolv.conf and unbound config. +UPGRADE_SCRIPT = r""" +#!/bin/bash +set -eux +export DEBIAN_FRONTEND=noninteractive +APT="apt-get -y -o DPkg::Lock::Timeout=300 \ +-o Dpkg::Options::=--force-confold \ +-o Dpkg::Options::=--force-confdef" +# --allow-releaseinfo-change is an update-only option, apt errors out if it +# is passed to install or full-upgrade +APT_UPDATE="$APT update --allow-releaseinfo-change" + +# Ensure we have the dpkg lock +systemctl disable --now unattended-upgrades \ + apt-daily.timer apt-daily-upgrade.timer || true +$APT purge unattended-upgrades || true + +# Backup our resolv.conf +systemctl disable --now systemd-resolved || true +systemctl mask systemd-resolved || true +cp /etc/resolv.conf /root/resolv.conf.pre-upgrade + +# Allow libc6 restarts +echo 'libraries/restart-without-asking boolean true' | debconf-set-selections + +# Upgrade to latest bookworm +$APT_UPDATE +$APT install debian-archive-keyring +$APT full-upgrade + +# Rewrite both sources.list and .sources +shopt -s nullglob +for f in /etc/apt/sources.list /etc/apt/sources.list.d/*.list \ + /etc/apt/sources.list.d/*.sources; do + sed -i 's/bookworm/trixie/g' "$f" +done +grep -rq trixie /etc/apt/sources.list /etc/apt/sources.list.d/ + +$APT_UPDATE +$APT full-upgrade +$APT --purge autoremove +apt-get clean + +cp /root/resolv.conf.pre-upgrade /etc/resolv.conf + +. /etc/os-release +test "$VERSION_ID" = "13" +""" + class DNSConfigurationError(Exception): """Raised on DNS reachability or response failure.""" @@ -425,6 +476,31 @@ def check(): f" Warning: Services on ports {ports} not ready after {timeout}s" ) + def upgrade_debian(self): + """In-place dist-upgrade this relay from Debian 12 to 13. + + Our apt-pin freezes dovecot at its bookworm build while the trixie + time_t transition removes libssl3/libtirpc3 underneath it, so dovecot + survives as an unrunnable package. Redeploy afterwards. + """ + path = "/root/cmlxc-upgrade.sh" + self.push_file_content(path, UPGRADE_SCRIPT, mode="755") + ret = self.out.shell(f"incus exec {self.name} -- {path}") + if ret: + raise SetupError(f"Debian upgrade failed on {self.shortname} (exit {ret})") + + self.out.print("Restarting container after upgrade ...") + restart = self.incus.run(["restart", self.name, "--timeout=120"], check=False) + if restart.returncode: + self.out.red("Graceful restart timed out; forcing") + self.stop(force=True) + self.start() + self.wait_ready() + if not self.verify_ssh(self.incus.ssh_config_path): + raise SetupError(f"{self.shortname}: no SSH after upgrade reboot") + release = self.bash('. /etc/os-release; echo "$PRETTY_NAME"') + self.out.green(f"{self.shortname} upgraded: {release}") + def verify_ssh(self, ssh_config): cmd = ["ssh", "-F", str(ssh_config), "-o", "ConnectTimeout=60"] cmd += [f"root@{self.domain}", "hostname"] diff --git a/tests/test_cli.py b/tests/test_cli.py index d245c14..de7ae45 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -4,6 +4,7 @@ import pytest +from cmlxc.cli import get_parser, upgrade_cmd from cmlxc.driver_base import SourceSpec, parse_source, validate_relay_name from cmlxc.driver_cmdeploy import get_ini_overrides @@ -61,3 +62,9 @@ def test_ini_overrides_disable_ipv6(): assert "disable_ipv6" not in get_ini_overrides("cm0.localchat") overrides = get_ini_overrides("cm0.localchat", disable_ipv6=True) assert overrides["disable_ipv6"] == "True" + + +def test_upgrade_parses_multiple_names(): + args = get_parser().parse_args(["dist-upgrade", "cm0", "cm1"]) + assert args.func is upgrade_cmd + assert args.names == ["cm0", "cm1"]