diff --git a/static/dashboard/CHANGELOG.md b/static/dashboard/CHANGELOG.md index ae2f80959..1d9e78b01 100644 --- a/static/dashboard/CHANGELOG.md +++ b/static/dashboard/CHANGELOG.md @@ -4,6 +4,7 @@ One line per version, newest first. Every REST-facing change bumps the version i (the app surfaces it via `/system/dashboard/update-check` and the "Update available" pill), so this file is the human record of what each bump enables. Keep entries short: `version - change (TICKET)`. +- **1.3.3** - Base-map job takes a file id, composes the URL (K2GO-394). The `basemaps` runner now accepts a bare pmtiles file name per item (`POST /api/basemaps/download {ids:[.pmtiles]}`) and composes `https://iiab.switnet.org/maps/2/` itself, instead of taking a full URL. This matches the kiwix split: the app holds the catalog and sends a light id, the box owns the mirror host -- so the switnet host never enters the app. The id is validated as a plain `.pmtiles` file name (no path, no traversal); archives (search) are not delegated here. No other behavior changes (same aria2 flags, reconnect loop, cancel-cleanup). (K2GO-394) - **1.3.2** - Base-map downloads via the durable job engine (K2GO-394). New `basemaps` job type + runner (`sockets/maps-base.exec.ts`), reached through the generic surface (`POST /api/basemaps/download {items:[]}`, `GET /api/basemaps/jobs/:id`, `POST /api/basemaps/jobs/:id/{cancel,pause,resume,retry}`). It downloads the selected global map pmtiles (vector / satellite / terrain) with aria2 straight into `/library/www/maps`, reusing the EXACT kiwix mechanism -- the canonical aria2 flag set plus the `withRetry` OUTER reconnect loop -- so a full Wi-Fi drop (aria2 exits on DNS, code 19) recovers by re-running aria2 which resumes via `--continue`, surfaced as "Reconnecting n/5". This replaces the aria2c the maps runrole ran IN-PROOT, which could not recover a mobile-radio drop (it wedged with no exit): the app runs this job (server up) and the runrole then only post-processes, its download tasks SKIPPING via `creates:` (see the maps role's `is_proot` delegate patch). Two things learned the hard way and encoded here: it passes the DIRECT pmtiles URL, NOT a `.meta4` metalink (metalink downloads were the exact case aria2 could not recover), and it does NOT diverge the flags (an aggressive `--max-tries=1 --timeout=10` cut wedged worse; the kiwix values are load-bearing). Kept SEPARATE from the FQR `maps` type (tile-extract). Device-verified: cut Wi-Fi at 27% of a pmtiles -> "Reconnecting 5/5", partial kept -> restore -> resumed to done, file complete at dest_path. (K2GO-394) - **1.3.1** - Live firehose signal for the app-side backstop (K2GO-386, ADR-386 §6). New read-only `GET /system/disk-guard/firehose` returns `{ recurring, maxStreak, paths, lastTruncatedAtMs, now }`. The in-box guard (1.3.0) truncates a runaway `.log` every tick, so the disk may never go low -- but a recurring firehose means an off-proot orphan the box CANNOT stop; only an app-side reap can. This endpoint exposes the guard's LIVE in-memory streak state (never a parsed log line, so a restart-resolved firehose reports clean) as the app's SECOND reap trigger. `recurring` is `maxStreak >= 2` (a single `.log` refilled past the cap on at least two consecutive ticks); `lastTruncatedAtMs` (wall-clock, 0 if never) lets the app judge freshness. It is an ALERT only: the app re-probes live log growth before it reaps (confirm before acting). Localhost-only. (K2GO-386) - **1.3.0** - Proot log rotation, dash-node-triggered (K2GO-386, ADR-386). proot has no systemd/cron, so `/etc/cron.daily/logrotate` never runs — logrotate was installed but never triggered, and a service log (php-fpm, dash-node) could grow until the device hit ENOSPC. dash-node now runs, every 10 min (no work at boot; `timer.unref`), a firehose guard THEN `logrotate /etc/logrotate.conf`: the guard truncates any log past ~1 GiB in place first (a runaway ~GB/min that logrotate would otherwise copy — doubling disk + pegging CPU on a weak phone), so L2 never meets a firehose; a recurring firehose is flagged for the future app-side reap (ADR-386 §6). The K2Go-owned config `/etc/logrotate.d/k2go` (copytruncate + `size 100M`, proot-correct — no reopen signal — overriding the RPi-oriented nginx/php-fpm snippets and adding calibre-web + dash-node; kiwix has no log, kolibri self-rotates) is installed at deploy by `tools/setup-proot-logging.sh` (rootfs build + rebuild/dev-push), not at boot. Not a REST-surface change; the version bump is the delivery mechanism for the new dash-node behavior (no ansible role yet). (K2GO-386) diff --git a/static/dashboard/package.json b/static/dashboard/package.json index 8b25f9249..f2c67742c 100644 --- a/static/dashboard/package.json +++ b/static/dashboard/package.json @@ -1,6 +1,6 @@ { "name": "dashboard-console", - "version": "1.3.2", + "version": "1.3.3", "description": "", "main": "index.js", "scripts": { diff --git a/static/dashboard/sockets/maps-base.exec.ts b/static/dashboard/sockets/maps-base.exec.ts index f9d1098c9..40a2c4213 100644 --- a/static/dashboard/sockets/maps-base.exec.ts +++ b/static/dashboard/sockets/maps-base.exec.ts @@ -20,9 +20,14 @@ import path from 'path'; // = maps_serve_path = dest_base_path in roles/maps; the runrole moves each pmtiles here. const MAPS_DIR = '/library/www/maps'; -// The app passes one catalog_file_url per selected layer. Only plain https URLs (no shell is -// used -- spawn takes an argv -- so this guards SSRF/log-shape, not shell injection). -const SAFE_URL = /^https:\/\/[^\s'"`$<>|;()]+$/; +// The mirror host, kept out of the app -- same split as kiwix (kiwix.exec.ts BASE_URL): the app +// sends a bare file id, the box composes the URL. K2GO-394. +const MAPS_BASE_URL = 'https://iiab.switnet.org/maps/2/'; +// The app sends one pmtiles file name per selected layer (the catalog id, e.g. +// "terrarium.2025-12-10.z00-z07.pmtiles"). Validate it as a plain file name -- no path, no +// traversal -- before composing the URL. No shell is used (spawn takes an argv), so this guards the +// URL shape / SSRF, not shell injection. Only .pmtiles: archives (search) are not delegated here. +const SAFE_ID = /^[A-Za-z0-9][A-Za-z0-9._-]*\.pmtiles$/; // ADFA-4832 CANONICAL aria2 flag set -- an EXACT copy of kiwix.exec.ts (only -d differs). The kiwix // runner recovers from a Wi-Fi drop (aria2 --max-tries=5 absorbs in-flight blips; a FULL interface @@ -100,22 +105,23 @@ function cleanupPartials(files: string[]): void { } const mapsBaseRunner: (ctx: RunnerContext) => Promise = async (ctx) => { - const urls = ctx.ids.map(String).filter((u) => u.length > 0); - if (urls.length === 0) throw new Error('no base-map URLs requested'); - for (const u of urls) if (!SAFE_URL.test(u)) throw new Error(`unsafe base-map URL: ${u}`); - // aria2 saves each URL under its basename in MAPS_DIR (no --out); the same basename the maps role - // expects at dest_path. Used to prune the right partial on cancel. - const files = urls.map((u) => path.basename(u)); + // The app sends bare pmtiles file ids (the catalog file names); the box composes the URL. + const files = ctx.ids.map(String).filter((u) => u.length > 0); + if (files.length === 0) throw new Error('no base-map files requested'); + for (const f of files) if (!SAFE_ID.test(f)) throw new Error(`unsafe base-map file id: ${f}`); + // Compose each id against the mirror base (same split as kiwix). aria2 saves each under its own + // name in MAPS_DIR (no --out), which is exactly the runrole's dest_path, so the role's `creates:` + // then skips the download. `files` are the on-disk names, used to prune the right partial on cancel. + const urls = files.map((f) => MAPS_BASE_URL + f); fs.mkdirSync(MAPS_DIR, { recursive: true }); ctx.throwIfCanceled(); ctx.update({ phase: 'downloading', speed: 0, detail: files.join(', ') }); // Pass the DIRECT pmtiles URL (not .meta4), the same way the kiwix runner passes the .zim - // URL directly. aria2 downloads it into MAPS_DIR under its own basename -- exactly the runrole's - // dest_path, so the role's `creates:` then skips the download. (An explicit .meta4 metalink is - // what the in-proot runrole used, and metalink downloads are what wedged aria2 on a network drop - // -- K2GO-394; --follow-metalink=mem still honors a metalink the mirror serves on its own.) + // URL directly. (An explicit .meta4 metalink is what the in-proot runrole used, and metalink + // downloads are what wedged aria2 on a network drop -- K2GO-394; --follow-metalink=mem still + // honors a metalink the mirror serves on its own.) try { await withRetry(() => new Promise((resolve, reject) => { const dl = ctx.spawn('/usr/bin/aria2c', [...ARIA2_ARGS, ...urls]); diff --git a/tools/upstream-patches/patches/0003-maps-download-is-proot-delegate.patch b/tools/upstream-patches/patches/0003-maps-download-is-proot-delegate.patch index 9790cc818..4cf83f92a 100644 --- a/tools/upstream-patches/patches/0003-maps-download-is-proot-delegate.patch +++ b/tools/upstream-patches/patches/0003-maps-download-is-proot-delegate.patch @@ -1,7 +1,7 @@ Upstream-PR: not yet submitted Upstream-Status: open Applies-to: roles/maps/tasks/download_large_file.yml -Summary: On proot, delegate the base-map pmtiles download to dash-node (the dashboard's durable job engine) instead of running a blocking aria2c in-proot. The in-proot aria2 could not recover from a full mobile-radio drop -- it wedged with no exit (K2GO-394). dash-node downloads the pmtiles with its own resumable, reconnecting aria2 (--continue plus an outer reconnect loop -- the proven kiwix mechanism) into the maps serve dir BEFORE this role runs. So the original download task is gated `when: not is_proot` (unchanged off proot), and a new `when: is_proot` step asserts the file is already at dest_path (a missing file means the host skipped the dash-node step); the role then only post-processes. Carried for our build; WIP for upstream (K2GO-394). +Summary: On proot, delegate the base-map PMTILES download to dash-node (the dashboard's durable job engine) instead of running a blocking aria2c in-proot. The in-proot aria2 could not recover from a full mobile-radio drop -- it wedged with no exit (K2GO-394). dash-node downloads the pmtiles with its own resumable, reconnecting aria2 (--continue plus an outer reconnect loop -- the proven kiwix mechanism) into the maps serve dir BEFORE this role runs. So a pmtiles download is gated `when: not is_proot` (unchanged off proot), and a new step asserts the file is already at dest_path. Archives (expand_archive, e.g. the search tarball) are NOT delegated -- dash-node does not extract them -- so they still download AND extract in-proot (the download task also does the extract/mv, and the search tarball is small so the wedge risk is low). The role then only post-processes the delegated pmtiles. Carried for our build; WIP for upstream (K2GO-394). diff --git a/roles/maps/tasks/download_large_file.yml b/roles/maps/tasks/download_large_file.yml index f8aa705..78a55cc 100644 @@ -11,26 +11,26 @@ index f8aa705..78a55cc 100644 args: executable: /bin/bash creates: "{{ dest_path }}" -+ when: not is_proot ++ when: not is_proot or (expand_archive | default(false)) + -+ # proot / Android (K2GO-394): dash-node -- the dashboard's durable job engine -- downloads the -+ # base-map pmtiles with its own resumable, reconnecting aria2 (--continue plus an outer reconnect -+ # loop) BEFORE this role runs. On proot this role only post-processes, so it must NOT download -+ # in-proot, where a blocking aria2 cannot recover from a mobile-radio drop (it wedges with no -+ # exit). The file is expected at dest_path already; fail early and clearly if the host skipped the -+ # dash-node step, rather than leaving a broken symlink for maps-update.py to trip on. ++ # proot / Android (K2GO-394): dash-node -- the durable job engine -- downloads the base-map ++ # PMTILES with its own resumable, reconnecting aria2 (the proven kiwix mechanism) BEFORE this ++ # role runs, because a blocking in-proot aria2 cannot recover a mobile-radio drop (it wedges with ++ # no exit). So a pmtiles download is skipped here and only asserted present. Archives ++ # (expand_archive, e.g. the search tarball) are NOT delegated -- dash-node does not extract them -- ++ # so the task above still downloads AND extracts them in-proot (small; the wedge risk is low). + - name: "Base map {{ file_name }} must be present on proot (downloaded by dash-node)" + stat: + path: "{{ dest_path }}" + register: proot_basemap -+ when: is_proot ++ when: is_proot and not (expand_archive | default(false)) + + - name: "Fail if dash-node did not place {{ file_name }} on proot" + assert: + that: proot_basemap.stat.exists + fail_msg: "On proot, {{ dest_path }} must be downloaded by dash-node before the maps role runs (K2GO-394)." + quiet: yes -+ when: is_proot ++ when: is_proot and not (expand_archive | default(false)) rescue: # We output summaries to a log file for the user's benefit,