diff --git a/controller/app/src/main/java/org/appdevforall/k2go/catalog/data/CatalogRefreshScheduler.java b/controller/app/src/main/java/org/appdevforall/k2go/catalog/data/CatalogRefreshScheduler.java index 17dd6dfc9..8b083446f 100644 --- a/controller/app/src/main/java/org/appdevforall/k2go/catalog/data/CatalogRefreshScheduler.java +++ b/controller/app/src/main/java/org/appdevforall/k2go/catalog/data/CatalogRefreshScheduler.java @@ -31,10 +31,15 @@ private CatalogRefreshScheduler() { } private static Data input(String name, String manifestUrl, String basename) { + return input(name, manifestUrl, basename, false); + } + + private static Data input(String name, String manifestUrl, String basename, boolean force) { return new Data.Builder() .putString(CatalogRefreshWorker.KEY_NAME, name) .putString(CatalogRefreshWorker.KEY_MANIFEST_URL, manifestUrl) .putString(CatalogRefreshWorker.KEY_BASENAME, basename) + .putBoolean(CatalogRefreshWorker.KEY_FORCE, force) .build(); } @@ -86,4 +91,22 @@ public static void refreshNow(Context ctx, String name, String manifestUrl, Stri .enqueueUniqueWork("catalog-refresh-now-" + name, ExistingWorkPolicy.KEEP, req); } + + /** + * K2GO-390: force an on-demand check that bypasses the worker's TTL gate (for a 404 self-heal -- + * the catalog may have rolled to a newer dated file within the TTL window). The ETag conditional + * GET still makes it cheap. Its own unique name (KEEP) coalesces a burst of failures into one run. + */ + public static void forceRefresh(Context ctx, String name, String manifestUrl, String basename) { + // K2GO-390: EXPEDITED so it dispatches promptly -- a 404 self-heal must land before the caller's + // bounded retries give up (falls back to a normal request if the expedited quota is spent). + OneTimeWorkRequest req = new OneTimeWorkRequest.Builder(CatalogRefreshWorker.class) + .setConstraints(constraints(NetworkType.CONNECTED)) + .setExpedited(androidx.work.OutOfQuotaPolicy.RUN_AS_NON_EXPEDITED_WORK_REQUEST) + .setInputData(input(name, manifestUrl, basename, true)) + .build(); + WorkManager.getInstance(ctx.getApplicationContext()) + .enqueueUniqueWork("catalog-refresh-force-" + name, + ExistingWorkPolicy.KEEP, req); + } } diff --git a/controller/app/src/main/java/org/appdevforall/k2go/catalog/data/CatalogRefreshWorker.java b/controller/app/src/main/java/org/appdevforall/k2go/catalog/data/CatalogRefreshWorker.java index 5d8969b1f..94dc94237 100644 --- a/controller/app/src/main/java/org/appdevforall/k2go/catalog/data/CatalogRefreshWorker.java +++ b/controller/app/src/main/java/org/appdevforall/k2go/catalog/data/CatalogRefreshWorker.java @@ -36,6 +36,9 @@ public final class CatalogRefreshWorker extends Worker { public static final String KEY_NAME = "name"; public static final String KEY_MANIFEST_URL = "manifest_url"; public static final String KEY_BASENAME = "basename"; + // K2GO-390: bypass the TTL gate for an on-demand check (e.g. a 404 self-heal needs to look now, + // even if the weekly check ran recently). The ETag conditional GET still keeps it cheap. + public static final String KEY_FORCE = "force"; public CatalogRefreshWorker(@NonNull Context context, @NonNull WorkerParameters params) { super(context, params); @@ -54,7 +57,8 @@ public Result doWork() { CatalogRefreshStore store = new CatalogRefreshStore(ctx); long now = System.currentTimeMillis(); - if (!CatalogFreshness.dueForCheck(store.lastCheckMs(name), now, CatalogFreshness.DEFAULT_TTL_MS)) { + boolean force = getInputData().getBoolean(KEY_FORCE, false); + if (!force && !CatalogFreshness.dueForCheck(store.lastCheckMs(name), now, CatalogFreshness.DEFAULT_TTL_MS)) { return Result.success(); // still fresh; do not hit the network } diff --git a/controller/app/src/main/java/org/appdevforall/k2go/redesign/ContentDownloadSession.java b/controller/app/src/main/java/org/appdevforall/k2go/redesign/ContentDownloadSession.java index 3c231f398..299a80cf4 100644 --- a/controller/app/src/main/java/org/appdevforall/k2go/redesign/ContentDownloadSession.java +++ b/controller/app/src/main/java/org/appdevforall/k2go/redesign/ContentDownloadSession.java @@ -39,6 +39,9 @@ public interface Host { void notify(String label); // update the foreground notification for the current item void stop(); // stopForeground(true) + stopSelf() void onItemDone(String key); // item confirmed DONE -> drop its wishlist entry (ADFA-4897) + // K2GO-390: item gave up (FAILED). The host may self-heal (refresh the catalog and re-resolve) + // and must bound retries so a stale/gone entry is not re-drained forever. Default no-op. + default void onItemError(String key) {} } private final String type; // "kiwix" / "books" -> /api/ @@ -235,7 +238,12 @@ private void startItem(final int i) { @Override public void onError(String message) { // ADFA-4893: server owns reconnection (visible); on give-up, FAILED for a manual Retry. android.util.Log.w("K2Go-Provision", "[" + type + "] job [" + i + "] error: " + message); - status[i] = FAILED; reconnectAttempt = 0; publish(); pump(); + status[i] = FAILED; reconnectAttempt = 0; publish(); + // K2GO-390: let the host self-heal (refresh the catalog, re-resolve) and bound retries, + // so a stale/gone item does not re-drain forever. + String k = key(i); + if (host != null && !k.isEmpty()) host.onItemError(k); + pump(); } }); } diff --git a/controller/app/src/main/java/org/appdevforall/k2go/redesign/KiwixCatalog.java b/controller/app/src/main/java/org/appdevforall/k2go/redesign/KiwixCatalog.java index dee1b7945..f1b9979e5 100644 --- a/controller/app/src/main/java/org/appdevforall/k2go/redesign/KiwixCatalog.java +++ b/controller/app/src/main/java/org/appdevforall/k2go/redesign/KiwixCatalog.java @@ -13,8 +13,9 @@ * background refresh (re-downloading an updated CSV) can be layered on later. * * Shape built in memory: - * { project: { lang: { "": {creator,flavour,size,date,file} } } } - * Files with no language token are bucketed under "mul" (language-agnostic). + * { project: { lang: { "": {creator,flavour,size,date,file} } } } + * The entry key joins creator and flavour with KEY_SEP (see below). Files with no language + * token are bucketed under "mul" (language-agnostic). * ============================================================================ */ package org.appdevforall.k2go.redesign; @@ -26,7 +27,14 @@ import org.json.JSONObject; +import org.appdevforall.k2go.catalog.data.CatalogOverlay; +import org.appdevforall.k2go.catalog.data.CatalogRefreshScheduler; +import org.appdevforall.k2go.config.DownloadEndpoints; + import java.io.BufferedReader; +import java.io.File; +import java.io.FileInputStream; +import java.io.InputStream; import java.io.InputStreamReader; import java.util.Iterator; import java.util.LinkedHashSet; @@ -38,6 +46,22 @@ private KiwixCatalog() {} private static final String TAG = "KiwixCatalog"; private static final String CSV_ASSET = "kiwix_catalog.csv"; + // ADFA-4849/K2GO-390: entry-key delimiter joining creator and flavour into the map key. A + // U+0001 control char is used because it can never appear in a creator or flavour token, so + // "" keys cannot collide across rows. The cart, wishlist and resolver all copy + // this key verbatim, so the delimiter stays internal. It is spelled out here (it used to be an + // invisible char inside "") so it is visible and greppable -- do not change it without migrating + // any persisted wishlist keys, which embed it. + private static final String KEY_SEP = "\u0001"; + + // K2GO-390 (ADR-390): the catalog is refreshed like Kolibri's -- a hosted manifest + overlay, + // ETag/hash-gated -- reusing the catalog-agnostic core. Flat, so no tree machinery. The overlay + // (when a newer CSV has been pulled) is preferred over the APK asset; the asset is the offline + // baseline. CATALOG name namespaces its refresh state; BASENAME is shared with the overlay so the + // worker writes exactly where loadCsv reads. + private static final String CATALOG_NAME = "kiwix"; + private static final String MANIFEST_URL = DownloadEndpoints.APK_REPO + "/catalogs/kiwix.manifest.json"; + /** Language-agnostic bucket (files whose name carries no language token, e.g. many videos). */ public static final String MUL = "mul"; @@ -47,27 +71,93 @@ public interface Listener { } private static volatile JSONObject inMemory; - - /** Loads the baked CSV (once per process) off the main thread; posts back on the main thread. */ + // Which source the cache came from: -1 = not loaded, 0 = APK asset, >0 = the overlay's lastModified. + private static volatile long cachedOverlayMtime = -1L; + + /** + * Loads the catalog (overlay if pulled, else the baked asset) off the main thread; posts back on the + * main thread. K2GO-390: also nudges the freshness refresh (weekly + an opportunistic TTL-gated + * check now, since the picker is opening). Both refreshes are network-constrained WorkManager jobs, + * so offline is a silent no-op -- the asset/overlay stays the offline baseline. See ADR-390. + */ public static void getOrFetch(Context context, Listener listener) { - JSONObject mem = inMemory; + final Context app = context.getApplicationContext(); + nudgeRefresh(app); + + JSONObject mem; + synchronized (KiwixCatalog.class) { + reloadIfOverlayChanged(app); // drop the cache if a newer overlay landed + mem = inMemory; + } if (mem != null) { post(() -> listener.onReady(mem)); return; } new Thread(() -> { - JSONObject db = loadCsv(context); - if (db != null && db.length() > 0) { - inMemory = db; - post(() -> listener.onReady(db)); - } else { - post(() -> listener.onError("Catalog unavailable")); + JSONObject db; + synchronized (KiwixCatalog.class) { // one loader wins; the rest reuse the cache + if (inMemory == null) inMemory = loadCsv(app); + db = inMemory; } + if (db != null && db.length() > 0) post(() -> listener.onReady(db)); + else post(() -> listener.onError("Catalog unavailable")); }).start(); } + // Nudge the freshness refresh once per process (K2GO-390): weekly (KEEP) + an opportunistic, + // TTL-gated check. Network-constrained, so offline is a no-op. A 404 forces its own check + // (forceRefresh), so this need not run on every catalog open (the drain opens it every ~2 s). + private static volatile boolean refreshNudged = false; + + private static void nudgeRefresh(Context app) { + if (refreshNudged) return; + refreshNudged = true; + CatalogRefreshScheduler.scheduleWeekly(app, CATALOG_NAME, MANIFEST_URL, CSV_ASSET); + CatalogRefreshScheduler.refreshNow(app, CATALOG_NAME, MANIFEST_URL, CSV_ASSET); + } + + /** + * K2GO-390: force a freshness check that bypasses the TTL gate. Called when a download 404s -- the + * catalog may have rolled to a newer dated file within the TTL window. Network-constrained, so + * offline is a silent no-op. Once the overlay lands, the next {@link #getOrFetch} adopts it and the + * drain re-resolves the (date-free) key to the current file. See ADR-390. + */ + public static void forceRefresh(Context context) { + CatalogRefreshScheduler.forceRefresh(context.getApplicationContext(), CATALOG_NAME, MANIFEST_URL, CSV_ASSET); + } + + /** + * K2GO-390: the current catalog version tag -- the overlay's mtime, or 0 for the baked asset. The + * self-heal counts failures against this ({@link ZimWishlist#bumpAttempts}): a refresh that replaces + * the overlay moves the tag and renews the retry budget; an unchanging catalog keeps it stable so the + * budget can reach its cap and drop a genuinely-gone item. Kept here so "which catalog version" has a + * single owner (the overlay basename lives only in this class). See ADR-390. + */ + public static long catalogVersionTag(Context context) { + File overlay = CatalogOverlay.file(context.getApplicationContext(), CSV_ASSET); + return overlay.exists() ? overlay.lastModified() : 0L; + } + + /** Drop the cache so the next load re-reads. K2GO-390: called after a refresh pulls a new overlay. */ + public static void invalidate() { + inMemory = null; + cachedOverlayMtime = -1L; + } + + /** If the overlay's mtime differs from what the cache was loaded from, drop the cache (ADR-390). */ + private static void reloadIfOverlayChanged(Context ctx) { + if (inMemory == null) return; + File overlay = CatalogOverlay.file(ctx, CSV_ASSET); + long mtime = overlay.exists() ? overlay.lastModified() : 0L; + if (mtime != cachedOverlayMtime) invalidate(); + } + private static JSONObject loadCsv(Context context) { JSONObject db = new JSONObject(); - try (BufferedReader r = new BufferedReader( - new InputStreamReader(context.getAssets().open(CSV_ASSET)))) { + // K2GO-390: prefer the pulled overlay over the APK asset; the asset is the offline baseline. + File overlay = CatalogOverlay.file(context, CSV_ASSET); + boolean useOverlay = overlay.exists(); + long mtime = useOverlay ? overlay.lastModified() : 0L; + try (InputStream in = useOverlay ? new FileInputStream(overlay) : context.getAssets().open(CSV_ASSET); + BufferedReader r = new BufferedReader(new InputStreamReader(in))) { String line; boolean header = true; while ((line = r.readLine()) != null) { @@ -95,12 +185,13 @@ private static JSONObject loadCsv(Context context) { v.put("size", bytes); v.put("date", date); v.put("file", file); - langObj.put(creator + "" + flavour, v); + langObj.put(creator + KEY_SEP + flavour, v); } } catch (Exception e) { Log.w(TAG, "kiwix_catalog.csv not read: " + e.getMessage()); return null; } + cachedOverlayMtime = mtime; // remember which source (asset=0 / overlay mtime) fed the cache return db; } diff --git a/controller/app/src/main/java/org/appdevforall/k2go/redesign/ZimDownloadService.java b/controller/app/src/main/java/org/appdevforall/k2go/redesign/ZimDownloadService.java index ab322782c..41b547b47 100644 --- a/controller/app/src/main/java/org/appdevforall/k2go/redesign/ZimDownloadService.java +++ b/controller/app/src/main/java/org/appdevforall/k2go/redesign/ZimDownloadService.java @@ -163,6 +163,29 @@ public int onStartCommand(Intent intent, int flags, int startId) { if (key != null && !key.isEmpty()) ZimWishlist.remove(getApplicationContext(), key); } + // K2GO-390: bound the self-heal. On a failure we force a catalog refresh; the drain then re-resolves + // the date-free key ("project|lang|flavour") to the current dated file and retries. After this many + // failed drains for one key, it is genuinely gone (or there is no fresh source) -- drop it so it + // stops re-draining. ~5 * the ~2s drain cadence gives the refresh time to land. See ADR-390. + private static final int MAX_HEAL_ATTEMPTS = 5; + + /** ADR-390: item gave up (likely a stale catalog -> 404). Force a freshness check and bound retries, + * so a rolled-over ZIM heals to its current file and a genuinely-gone one stops looping. */ + @Override public void onItemError(String key) { + if (key == null || key.isEmpty()) return; + Context app = getApplicationContext(); + KiwixCatalog.forceRefresh(app); // network-constrained; offline is a silent no-op + // Count failures against the current catalog version (KiwixCatalog owns what that is). A refresh + // that changes the catalog resets the budget (see ZimWishlist.bumpAttempts); only an unchanging + // catalog climbs to the cap = genuinely gone / no fresh source. + int attempts = ZimWishlist.bumpAttempts(app, key, KiwixCatalog.catalogVersionTag(app)); + if (attempts >= MAX_HEAL_ATTEMPTS) { + android.util.Log.w("K2Go-Provision", + "kiwix item still failing after " + attempts + " attempts; dropping " + key); + ZimWishlist.remove(app, key); + } + } + private void createNotificationChannel() { if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O) { NotificationChannel channel = new NotificationChannel( diff --git a/controller/app/src/main/java/org/appdevforall/k2go/redesign/ZimWishlist.java b/controller/app/src/main/java/org/appdevforall/k2go/redesign/ZimWishlist.java index 9b49a7f32..b6dc1c322 100644 --- a/controller/app/src/main/java/org/appdevforall/k2go/redesign/ZimWishlist.java +++ b/controller/app/src/main/java/org/appdevforall/k2go/redesign/ZimWishlist.java @@ -79,4 +79,28 @@ public static void remove(Context ctx, String key) { public static void clear(Context ctx) { prefs(ctx).edit().remove(KEY).apply(); } + + /** + * K2GO-390: record one failed download attempt for a key, counted AGAINST a catalog version + * ({@code catalogTag} = the overlay's mtime, 0 for the asset). If the catalog changed since the last + * failure, the count RESETS to 1 -- a refreshed catalog gives the current file a fresh budget; only + * failures against an unchanging catalog climb toward the cap (= genuinely gone, or no fresh source). + * The count rides in the entry; a confirmed DONE removes the entry via {@link #remove}. Returns the + * new count, or 0 if the key is absent. + */ + public static int bumpAttempts(Context ctx, String key, long catalogTag) { + if (key == null) return 0; + JSONArray cur = all(ctx); + int count = 0; + for (int i = 0; i < cur.length(); i++) { + JSONObject o = cur.optJSONObject(i); + if (o != null && key.equals(o.optString("key"))) { + count = (o.optLong("catTag", Long.MIN_VALUE) == catalogTag) ? o.optInt("attempts", 0) + 1 : 1; + try { o.put("attempts", count).put("catTag", catalogTag); } catch (Exception ignored) {} + break; + } + } + prefs(ctx).edit().putString(KEY, cur.toString()).apply(); + return count; + } } diff --git a/controller/app/tools/build_kiwix_catalog.py b/controller/app/tools/build_kiwix_catalog.py index 1b70f4a38..72c94a761 100644 --- a/controller/app/tools/build_kiwix_catalog.py +++ b/controller/app/tools/build_kiwix_catalog.py @@ -12,6 +12,8 @@ # Usage: # python3 build_kiwix_catalog.py # fetch live -> CSV # python3 build_kiwix_catalog.py --from-file F # parse a saved dump (===CATEGORY markers), .txt or .gz +# python3 build_kiwix_catalog.py --manifest kiwix.manifest.json --csv-url +# # also emit the freshness manifest (K2GO-390 ops path) # # Optional: `pip install pycountry` for the full ISO language set; otherwise an # embedded set is used (covers the current catalog). @@ -19,7 +21,8 @@ # Gradle runs this only on release builds (assembleRelease/bundleRelease); run it # manually any time with `./gradlew refreshKiwixCatalog`. # ============================================================================ -import argparse, csv, gzip, os, re, sys, urllib.request +import argparse, csv, gzip, hashlib, json, os, re, sys, urllib.request +from datetime import datetime, timezone CATS = ["devdocs","freecodecamp","gutenberg","ifixit","libretexts","maps","mooc", "other","phet","psiram","stack_exchange","ted","videos","vikidia","wikibooks", @@ -39,6 +42,11 @@ WIKI_FAMILY = {"wikipedia","wiktionary","wikibooks","wikiquote","wikisource", "wikiversity","wikivoyage","wikinews","vikidia"} +# K2GO-390: kiwix re-releases within a month add a letter (2026-07a, 2026-07f). The letter is part +# of the DATE, not the flavour -- keep it here (one source) so both date detections agree, or the +# suffix leaks into the flavour and the self-heal key (creator+flavour) drifts across a roll-over. +DATE_RE = re.compile(r"\d{4}-\d{2}[a-z]?") + try: import pycountry ISO1 = {l.alpha_2.lower() for l in pycountry.languages if hasattr(l, "alpha_2")} @@ -88,14 +96,14 @@ def parse_name(fn, category=""): stem = fn[:-4] if fn.lower().endswith(".zim") else fn toks = stem.split("_") date = "" - if re.fullmatch(r"\d{4}-\d{2}", toks[-1]): + if DATE_RE.fullmatch(toks[-1]): date = toks[-1]; toks = toks[:-1] creator = toks[0] if toks else stem mids = toks[1:] lang, idx = "", -1 if category in WIKI_FAMILY and mids and norm(mids[0]) not in BLACKLIST \ - and not re.fullmatch(r"\d{4}-\d{2}", mids[0]): + and not DATE_RE.fullmatch(mids[0]): lang, idx = norm(mids[0]), 0 # trust the strict wiki grammar else: if mids: @@ -156,6 +164,13 @@ def main(): ap = argparse.ArgumentParser() ap.add_argument("--from-file") ap.add_argument("--out") + # K2GO-390 (ADR-390): also emit a manifest so the app can refresh the catalog (ETag/hash-gated), + # mirroring Kolibri's catalogs/kolibri.manifest.json. Ops runs the generator with --manifest and + # --csv-url, then uploads the CSV + manifest to APK_REPO/catalogs/. Omit for a plain asset refresh. + ap.add_argument("--manifest", help="path to write kiwix.manifest.json (enables manifest emission)") + ap.add_argument("--csv-url", + default="https://k2go-download.appdevforall.org/catalogs/kiwix_catalog.csv", + help="hosted URL of the CSV, recorded in the manifest") args = ap.parse_args() here = os.path.dirname(os.path.abspath(__file__)) @@ -178,6 +193,20 @@ def main(): w.writerow(["category","creator","lang","flavour","bytes","date","file"]) w.writerows(dedup) sys.stderr.write(f"wrote {out_path}: {len(dedup)} items (from {len(rows)} files)\n") + + if args.manifest: + with open(out_path, "rb") as f: + digest = hashlib.sha256(f.read()).hexdigest() + generated = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") + manifest = { + "hash": "sha256:" + digest, # the app verifies the downloaded CSV against this + "url": args.csv_url, # where the app pulls the refreshed CSV + "version": generated, # human/log label + "generated": generated, + } + with open(args.manifest, "w", encoding="utf-8") as f: + json.dump(manifest, f, indent=2) + sys.stderr.write(f"wrote {args.manifest}: {manifest['hash']}\n") return 0 if __name__ == "__main__": diff --git a/controller/docs/ADR-390-kiwix-catalog-freshness.md b/controller/docs/ADR-390-kiwix-catalog-freshness.md new file mode 100644 index 000000000..55798a1ef --- /dev/null +++ b/controller/docs/ADR-390-kiwix-catalog-freshness.md @@ -0,0 +1,127 @@ +# ADR-390: Kiwix self-healing catalog (offline-first, freshness on demand) + +Status: Accepted (implemented and device-verified 2026-09-05) +Ticket: K2GO-390 + +## 1. Problem + +The Kiwix ZIM catalog ships as a fixed asset, `assets/kiwix_catalog.csv`, generated off-device. It has +no freshness mechanism. Kiwix rotates ZIM files monthly and prunes old dates, so a baked entry that +points at a pruned date returns HTTP 404. + +Device-proven (K2GO-390, "Infinite loop attempting to download content"): a stale entry -> +`aria2 exited with code 3` (404) -> the item is never removed from the durable wishlist -> the +post-install provisioning drain re-attempts it about every 2 s, forever. Symptoms: CPU churn, +`iiab-make-kiwix-lib` re-running each cycle, `kiwix-serve` flapping, the box never settling. + +## 2. Facts that shape the design + +- Kiwix is FLAT: `category/file.zim`, two levels. (Kolibri is N-level -- 2 to 6, per category -- which + is why Kolibri needs a heavy topic-tree bundle and recursive browsing. Kiwix needs none of that.) +- The ZIM filename is `___.zim`. Only the date rolls over; the identity + (`creator`/`lang`/`flavour`) is stable. Confirmed live: `wikipedia_ab_all_maxi_2026-04` and `..._2026-07` + coexist on the server. A within-month re-release appends a letter (e.g. `2026-07a`, `2026-07f`); the + date is therefore `YYYY-MM[a-z]?`, and the generator must keep that letter on the DATE, not the flavour, + so the identity stays stable across a re-release (see `build_kiwix_catalog.py` `DATE_RE`). +- The wishlist key is ALREADY date-free: `project|lang|` (the third segment is the + catalog entry key -- `creator` joined to `flavour`, no date). So the same selection re-resolves to the + current dated file once the catalog is refreshed. This is what makes per-element self-heal cheap. +- The size delta between months is small (well under 10 %: e.g. 111G -> 115G, 2.0G -> 2.1G). It is not a + re-consent concern -- it is the same content, the next month's build. +- The generator already exists: `controller/app/tools/build_kiwix_catalog.py` (baked at + `assembleRelease`). Kolibri already has a catalog-agnostic freshness CORE (manifest + ETag + overlay + + store + scheduler, ADFA-5094) that Kiwix can reuse. + +## 3. Goals and constraints + +- Respect the upstream servers (Kiwix, Kolibri, Gutenberg): asset-first, minimal requests, conditional + (ETag) checks, no continuous scraping. Shipping the catalog as an asset exists to reduce requests. +- Offline-first: the asset is the guaranteed baseline. Browsing and selection work with no network. + Freshness is best-effort and connectivity-guarded. +- Easy for the user: silent background recovery, no UI, no size re-confirmation. +- Integral, not patched by parts. + +## 4. Decision + +A self-healing flat catalog for Kiwix that reuses the LIGHT freshness core and adds nothing tree-shaped. + +- **Asset baseline (kept).** `assets/kiwix_catalog.csv` still ships and still loads first. It is the + offline baseline and the instant-open source. +- **Overlay.** When online and the manifest changed, download the fresh CSV as an overlay + (`CatalogOverlay`), preferred over the asset -- mirror `BundledCatalogSource` (overlay-by-mtime). +- **Hybrid trigger (option a).** + - *At catalog open:* a conditional check (`If-None-Match` / ETag) against a small hosted manifest, + TTL-gated. `304` = match = no work (nothing is consumed). Changed = adopt the fresh CSV. + - *On a download 404:* force a conditional refresh. Because the wishlist key is date-free, it + re-resolves to the current dated file; retry against it. If the catalog did NOT change (same file), + the failure is real -> stop bounded (no loop). +- **Self-heal write-back.** Adopting the overlay heals the failing element AND every other stale one at + once. The refresh itself is the classifier: a changed file means "was stale, heal"; an unchanged file + means "genuinely gone or a network fault, stop". +- **Connectivity-guarded.** No network -> skip silently, use the asset/overlay, never error. Reuse the + existing `hasInternet` gating pattern (the rootfs path already does this). +- **No UI, no size re-confirm.** Recovery is in the background; the size delta is negligible. + +## 5. What we reuse, and what we do NOT + +- REUSE (catalog-agnostic, flat): `CatalogManifestClient` (ETag/304), `CatalogRefreshWorker` + (TTL/fetch/hash-verify/apply), `CatalogRefreshScheduler` (`scheduleWeekly` + `refreshNow`), + `CatalogRefreshStore` (per-catalog etag/hash/last-check), `CatalogOverlay`. Wire a `kiwix` catalog the + way `kolibri/data/CatalogRepositoryImpl.java` wires the Kolibri ones. +- DO NOT reuse: Kolibri's N-level topic-tree bundle and recursive browsing (`BundledTreeSource`, the + tree manifest). Kiwix is flat and needs none of it. This is the "simplify much more" for Kiwix. + +## 6. The loop fix falls out + +- Today `redesign/ContentDownloadSession.java` `onError` sets an item FAILED but never removes it from + `ZimWishlist` (removal happens only on DONE, `redesign/ZimDownloadService.java` `onItemDone`), so + `redesign/ZimProvisioner.java` `drain` re-hands the same stale key forever. +- New flow: a download failure triggers refresh-and-re-resolve. A stale item heals and retries against + the current file. Failures are counted against the catalog version (`ZimWishlist.bumpAttempts`, + keyed by the overlay mtime): a changed catalog renews the budget; after `MAX_HEAL_ATTEMPTS` (5) + failures against an UNCHANGED catalog the item is genuinely gone and is removed (leaves the wishlist) + so the drain stops. No infinite re-drain. + +## 7. Hosting (ops, required for the heal) -- LIVE + +The heal needs a fresh source: a Kiwix catalog manifest plus the refreshed CSV at +`APK_REPO + /catalogs/kiwix.manifest.json` (+ the CSV), mirroring `kolibri.manifest.json`. +`APK_REPO = https://k2go-download.appdevforall.org`. + +This is published and working. Device-verified 2026-09-05: the app pulled a fresh overlay +(`kiwix.manifest.json` generated 2026-09-02, ETag/hash recorded in the refresh store) and a 404 +self-healed end to end. If the source is ever absent the app degrades to asset-only (offline-first still +works) and a 404 cannot heal, so the published source must be kept fresh. + +Ops action per release cycle: regenerate and re-publish with the CURRENT generator +(`build_kiwix_catalog.py --manifest --csv-url ...`). The generator is the fix site -- re-running it +collapses re-release-letter entries (the `DATE_RE` fix) so they can heal; this is not a manual data edit. + +## 8. Lifecycle (who writes it, who clears it, what if it is missing) + +- Overlay: written by the refresh worker, preferred by the source, superseded by a newer overlay + (mtime/hash), removable (falls back to the asset). The store keeps per-catalog etag/hash/last-check; + namespaced by catalog name, so it already serves "kolibri, kiwix, ...". +- Wishlist: an item leaves on DONE (existing) or on a confirmed-gone item (new). A stale item heals + instead of looping. + +## 9. Verification + +- Unit: the pure freshness rules are already covered (`CatalogFreshness`). Unit coverage of the Kiwix + wiring and the stale-key re-resolve is a follow-up (not added in this change). +- Device (2026-09-05, OnePlus arm64, API 35), reproduced with a real stale entry + (`bulbagarden_en_all_nopic_2026-07`, 404 on the mirror): + - Pre-fix build: the 404 re-drains forever (35+ attempts, never removed). + - Heal: 404 -> forced refresh pulls the overlay -> the date-free key re-resolves to the live + `..._2026-05` -> downloads. Wishlist entry removed on DONE. + - Bounded (no newer version to adopt): 404 -> exactly `MAX_HEAL_ATTEMPTS` (5) attempts -> the item is + dropped -> the loop stops. + - Offline: the asset catalog works with no network and no error. + +## 10. Consequences + +- Stale catalogs self-heal silently; the infinite loop is gone; the upstream servers are respected; the + app still works offline. +- One more published artifact (the kiwix manifest + CSV) to maintain, mirroring Kolibri. +- The freshness core is confirmed reusable across content sources (Kolibri today, Kiwix here, Gutenberg + later) without dragging in any source's browsing shape.