K2GO-390 fix(kiwix): self-healing catalog to end the stale-download loop

controller/app/src/main/java/org/appdevforall/k2go/catalog/data/CatalogRefreshScheduler.java

@@ -31,10 +31,15 @@ private CatalogRefreshScheduler() {
     }
 
     private static Data input(String name, String manifestUrl, String basename) {
+        return input(name, manifestUrl, basename, false);
+    }
+
+    private static Data input(String name, String manifestUrl, String basename, boolean force) {
         return new Data.Builder()
                 .putString(CatalogRefreshWorker.KEY_NAME, name)
                 .putString(CatalogRefreshWorker.KEY_MANIFEST_URL, manifestUrl)
                 .putString(CatalogRefreshWorker.KEY_BASENAME, basename)
+                .putBoolean(CatalogRefreshWorker.KEY_FORCE, force)
                 .build();
     }
 
@@ -86,4 +91,22 @@ public static void refreshNow(Context ctx, String name, String manifestUrl, Stri
                 .enqueueUniqueWork("catalog-refresh-now-" + name,
                         ExistingWorkPolicy.KEEP, req);
     }
+
+    /**
+     * K2GO-390: force an on-demand check that bypasses the worker's TTL gate (for a 404 self-heal --
+     * the catalog may have rolled to a newer dated file within the TTL window). The ETag conditional
+     * GET still makes it cheap. Its own unique name (KEEP) coalesces a burst of failures into one run.
+     */
+    public static void forceRefresh(Context ctx, String name, String manifestUrl, String basename) {
+        // K2GO-390: EXPEDITED so it dispatches promptly -- a 404 self-heal must land before the caller's
+        // bounded retries give up (falls back to a normal request if the expedited quota is spent).
+        OneTimeWorkRequest req = new OneTimeWorkRequest.Builder(CatalogRefreshWorker.class)
+                .setConstraints(constraints(NetworkType.CONNECTED))
+                .setExpedited(androidx.work.OutOfQuotaPolicy.RUN_AS_NON_EXPEDITED_WORK_REQUEST)
+                .setInputData(input(name, manifestUrl, basename, true))
+                .build();
+        WorkManager.getInstance(ctx.getApplicationContext())
+                .enqueueUniqueWork("catalog-refresh-force-" + name,
+                        ExistingWorkPolicy.KEEP, req);
+    }
 }

controller/app/src/main/java/org/appdevforall/k2go/catalog/data/CatalogRefreshWorker.java

@@ -36,6 +36,9 @@ public final class CatalogRefreshWorker extends Worker {
     public static final String KEY_NAME = "name";
     public static final String KEY_MANIFEST_URL = "manifest_url";
     public static final String KEY_BASENAME = "basename";
+    // K2GO-390: bypass the TTL gate for an on-demand check (e.g. a 404 self-heal needs to look now,
+    // even if the weekly check ran recently). The ETag conditional GET still keeps it cheap.
+    public static final String KEY_FORCE = "force";
 
     public CatalogRefreshWorker(@NonNull Context context, @NonNull WorkerParameters params) {
         super(context, params);
@@ -54,7 +57,8 @@ public Result doWork() {
 
         CatalogRefreshStore store = new CatalogRefreshStore(ctx);
         long now = System.currentTimeMillis();
-        if (!CatalogFreshness.dueForCheck(store.lastCheckMs(name), now, CatalogFreshness.DEFAULT_TTL_MS)) {
+        boolean force = getInputData().getBoolean(KEY_FORCE, false);
+        if (!force && !CatalogFreshness.dueForCheck(store.lastCheckMs(name), now, CatalogFreshness.DEFAULT_TTL_MS)) {
             return Result.success();   // still fresh; do not hit the network
         }
 

controller/app/src/main/java/org/appdevforall/k2go/redesign/ContentDownloadSession.java

@@ -39,6 +39,9 @@ public interface Host {
         void notify(String label);     // update the foreground notification for the current item
         void stop();                   // stopForeground(true) + stopSelf()
         void onItemDone(String key);   // item confirmed DONE -> drop its wishlist entry (ADFA-4897)
+        // K2GO-390: item gave up (FAILED). The host may self-heal (refresh the catalog and re-resolve)
+        // and must bound retries so a stale/gone entry is not re-drained forever. Default no-op.
+        default void onItemError(String key) {}
     }
 
     private final String type;                          // "kiwix" / "books" -> /api/<type>
@@ -235,7 +238,12 @@ private void startItem(final int i) {
             @Override public void onError(String message) {
                 // ADFA-4893: server owns reconnection (visible); on give-up, FAILED for a manual Retry.
                 android.util.Log.w("K2Go-Provision", "[" + type + "] job [" + i + "] error: " + message);
-                status[i] = FAILED; reconnectAttempt = 0; publish(); pump();
+                status[i] = FAILED; reconnectAttempt = 0; publish();
+                // K2GO-390: let the host self-heal (refresh the catalog, re-resolve) and bound retries,
+                // so a stale/gone item does not re-drain forever.
+                String k = key(i);
+                if (host != null && !k.isEmpty()) host.onItemError(k);
+                pump();
             }
         });
     }

controller/app/src/main/java/org/appdevforall/k2go/redesign/KiwixCatalog.java

@@ -13,8 +13,9 @@
  *               background refresh (re-downloading an updated CSV) can be layered on later.
  *
  *               Shape built in memory:
- *                 { project: { lang: { "<creator>
<flavour>": {creator,flavour,size,date,file} } } }
- *               Files with no language token are bucketed under "mul" (language-agnostic).
+ *                 { project: { lang: { "<creator><KEY_SEP><flavour>": {creator,flavour,size,date,file} } } }
+ *               The entry key joins creator and flavour with KEY_SEP (see below). Files with no language
+ *               token are bucketed under "mul" (language-agnostic).
  * ============================================================================
  */
 package org.appdevforall.k2go.redesign;
@@ -26,7 +27,14 @@
 
 import org.json.JSONObject;
 
+import org.appdevforall.k2go.catalog.data.CatalogOverlay;
+import org.appdevforall.k2go.catalog.data.CatalogRefreshScheduler;
+import org.appdevforall.k2go.config.DownloadEndpoints;
+
 import java.io.BufferedReader;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.InputStream;
 import java.io.InputStreamReader;
 import java.util.Iterator;
 import java.util.LinkedHashSet;
@@ -38,6 +46,22 @@ private KiwixCatalog() {}
     private static final String TAG = "KiwixCatalog";
     private static final String CSV_ASSET = "kiwix_catalog.csv";
 
+    // ADFA-4849/K2GO-390: entry-key delimiter joining creator and flavour into the map key. A
+    // U+0001 control char is used because it can never appear in a creator or flavour token, so
+    // "<creator><flavour>" keys cannot collide across rows. The cart, wishlist and resolver all copy
+    // this key verbatim, so the delimiter stays internal. It is spelled out here (it used to be an
+    // invisible char inside "") so it is visible and greppable -- do not change it without migrating
+    // any persisted wishlist keys, which embed it.
+    private static final String KEY_SEP = "\u0001";
+
+    // K2GO-390 (ADR-390): the catalog is refreshed like Kolibri's -- a hosted manifest + overlay,
+    // ETag/hash-gated -- reusing the catalog-agnostic core. Flat, so no tree machinery. The overlay
+    // (when a newer CSV has been pulled) is preferred over the APK asset; the asset is the offline
+    // baseline. CATALOG name namespaces its refresh state; BASENAME is shared with the overlay so the
+    // worker writes exactly where loadCsv reads.
+    private static final String CATALOG_NAME = "kiwix";
+    private static final String MANIFEST_URL = DownloadEndpoints.APK_REPO + "/catalogs/kiwix.manifest.json";
+
     /** Language-agnostic bucket (files whose name carries no language token, e.g. many videos). */
     public static final String MUL = "mul";
 
@@ -47,27 +71,93 @@ public interface Listener {
     }
 
     private static volatile JSONObject inMemory;
-
-    /** Loads the baked CSV (once per process) off the main thread; posts back on the main thread. */
+    // Which source the cache came from: -1 = not loaded, 0 = APK asset, >0 = the overlay's lastModified.
+    private static volatile long cachedOverlayMtime = -1L;
+
+    /**
+     * Loads the catalog (overlay if pulled, else the baked asset) off the main thread; posts back on the
+     * main thread. K2GO-390: also nudges the freshness refresh (weekly + an opportunistic TTL-gated
+     * check now, since the picker is opening). Both refreshes are network-constrained WorkManager jobs,
+     * so offline is a silent no-op -- the asset/overlay stays the offline baseline. See ADR-390.
+     */
     public static void getOrFetch(Context context, Listener listener) {
-        JSONObject mem = inMemory;
+        final Context app = context.getApplicationContext();
+        nudgeRefresh(app);
+
+        JSONObject mem;
+        synchronized (KiwixCatalog.class) {
+            reloadIfOverlayChanged(app);   // drop the cache if a newer overlay landed
+            mem = inMemory;
+        }
         if (mem != null) { post(() -> listener.onReady(mem)); return; }
 
         new Thread(() -> {
-            JSONObject db = loadCsv(context);
-            if (db != null && db.length() > 0) {
-                inMemory = db;
-                post(() -> listener.onReady(db));
-            } else {
-                post(() -> listener.onError("Catalog unavailable"));
+            JSONObject db;
+            synchronized (KiwixCatalog.class) {   // one loader wins; the rest reuse the cache
+                if (inMemory == null) inMemory = loadCsv(app);
+                db = inMemory;
             }
+            if (db != null && db.length() > 0) post(() -> listener.onReady(db));
+            else post(() -> listener.onError("Catalog unavailable"));
         }).start();
     }
 
+    // Nudge the freshness refresh once per process (K2GO-390): weekly (KEEP) + an opportunistic,
+    // TTL-gated check. Network-constrained, so offline is a no-op. A 404 forces its own check
+    // (forceRefresh), so this need not run on every catalog open (the drain opens it every ~2 s).
+    private static volatile boolean refreshNudged = false;
+
+    private static void nudgeRefresh(Context app) {
+        if (refreshNudged) return;
+        refreshNudged = true;
+        CatalogRefreshScheduler.scheduleWeekly(app, CATALOG_NAME, MANIFEST_URL, CSV_ASSET);
+        CatalogRefreshScheduler.refreshNow(app, CATALOG_NAME, MANIFEST_URL, CSV_ASSET);
+    }
+
+    /**
+     * K2GO-390: force a freshness check that bypasses the TTL gate. Called when a download 404s -- the
+     * catalog may have rolled to a newer dated file within the TTL window. Network-constrained, so
+     * offline is a silent no-op. Once the overlay lands, the next {@link #getOrFetch} adopts it and the
+     * drain re-resolves the (date-free) key to the current file. See ADR-390.
+     */
+    public static void forceRefresh(Context context) {
+        CatalogRefreshScheduler.forceRefresh(context.getApplicationContext(), CATALOG_NAME, MANIFEST_URL, CSV_ASSET);
+    }
+
+    /**
+     * K2GO-390: the current catalog version tag -- the overlay's mtime, or 0 for the baked asset. The
+     * self-heal counts failures against this ({@link ZimWishlist#bumpAttempts}): a refresh that replaces
+     * the overlay moves the tag and renews the retry budget; an unchanging catalog keeps it stable so the
+     * budget can reach its cap and drop a genuinely-gone item. Kept here so "which catalog version" has a
+     * single owner (the overlay basename lives only in this class). See ADR-390.
+     */
+    public static long catalogVersionTag(Context context) {
+        File overlay = CatalogOverlay.file(context.getApplicationContext(), CSV_ASSET);
+        return overlay.exists() ? overlay.lastModified() : 0L;
+    }
+
+    /** Drop the cache so the next load re-reads. K2GO-390: called after a refresh pulls a new overlay. */
+    public static void invalidate() {
+        inMemory = null;
+        cachedOverlayMtime = -1L;
+    }
+
+    /** If the overlay's mtime differs from what the cache was loaded from, drop the cache (ADR-390). */
+    private static void reloadIfOverlayChanged(Context ctx) {
+        if (inMemory == null) return;
+        File overlay = CatalogOverlay.file(ctx, CSV_ASSET);
+        long mtime = overlay.exists() ? overlay.lastModified() : 0L;
+        if (mtime != cachedOverlayMtime) invalidate();
+    }
+
     private static JSONObject loadCsv(Context context) {
         JSONObject db = new JSONObject();
-        try (BufferedReader r = new BufferedReader(
-                new InputStreamReader(context.getAssets().open(CSV_ASSET)))) {
+        // K2GO-390: prefer the pulled overlay over the APK asset; the asset is the offline baseline.
+        File overlay = CatalogOverlay.file(context, CSV_ASSET);
+        boolean useOverlay = overlay.exists();
+        long mtime = useOverlay ? overlay.lastModified() : 0L;
+        try (InputStream in = useOverlay ? new FileInputStream(overlay) : context.getAssets().open(CSV_ASSET);
+                BufferedReader r = new BufferedReader(new InputStreamReader(in))) {
             String line;
             boolean header = true;
             while ((line = r.readLine()) != null) {
@@ -95,12 +185,13 @@ private static JSONObject loadCsv(Context context) {
                 v.put("size", bytes);
                 v.put("date", date);
                 v.put("file", file);
-                langObj.put(creator + "
" + flavour, v);
+                langObj.put(creator + KEY_SEP + flavour, v);
             }
         } catch (Exception e) {
             Log.w(TAG, "kiwix_catalog.csv not read: " + e.getMessage());
             return null;
         }
+        cachedOverlayMtime = mtime;   // remember which source (asset=0 / overlay mtime) fed the cache
         return db;
     }
 

controller/app/src/main/java/org/appdevforall/k2go/redesign/ZimDownloadService.java

@@ -163,6 +163,29 @@ public int onStartCommand(Intent intent, int flags, int startId) {
         if (key != null && !key.isEmpty()) ZimWishlist.remove(getApplicationContext(), key);
     }
 
+    // K2GO-390: bound the self-heal. On a failure we force a catalog refresh; the drain then re-resolves
+    // the date-free key ("project|lang|flavour") to the current dated file and retries. After this many
+    // failed drains for one key, it is genuinely gone (or there is no fresh source) -- drop it so it
+    // stops re-draining. ~5 * the ~2s drain cadence gives the refresh time to land. See ADR-390.
+    private static final int MAX_HEAL_ATTEMPTS = 5;
+
+    /** ADR-390: item gave up (likely a stale catalog -> 404). Force a freshness check and bound retries,
+     *  so a rolled-over ZIM heals to its current file and a genuinely-gone one stops looping. */
+    @Override public void onItemError(String key) {
+        if (key == null || key.isEmpty()) return;
+        Context app = getApplicationContext();
+        KiwixCatalog.forceRefresh(app);   // network-constrained; offline is a silent no-op
+        // Count failures against the current catalog version (KiwixCatalog owns what that is). A refresh
+        // that changes the catalog resets the budget (see ZimWishlist.bumpAttempts); only an unchanging
+        // catalog climbs to the cap = genuinely gone / no fresh source.
+        int attempts = ZimWishlist.bumpAttempts(app, key, KiwixCatalog.catalogVersionTag(app));
+        if (attempts >= MAX_HEAL_ATTEMPTS) {
+            android.util.Log.w("K2Go-Provision",
+                    "kiwix item still failing after " + attempts + " attempts; dropping " + key);
+            ZimWishlist.remove(app, key);
+        }
+    }
+
     private void createNotificationChannel() {
         if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O) {
             NotificationChannel channel = new NotificationChannel(

controller/app/src/main/java/org/appdevforall/k2go/redesign/ZimWishlist.java

@@ -79,4 +79,28 @@ public static void remove(Context ctx, String key) {
     public static void clear(Context ctx) {
         prefs(ctx).edit().remove(KEY).apply();
     }
+
+    /**
+     * K2GO-390: record one failed download attempt for a key, counted AGAINST a catalog version
+     * ({@code catalogTag} = the overlay's mtime, 0 for the asset). If the catalog changed since the last
+     * failure, the count RESETS to 1 -- a refreshed catalog gives the current file a fresh budget; only
+     * failures against an unchanging catalog climb toward the cap (= genuinely gone, or no fresh source).
+     * The count rides in the entry; a confirmed DONE removes the entry via {@link #remove}. Returns the
+     * new count, or 0 if the key is absent.
+     */
+    public static int bumpAttempts(Context ctx, String key, long catalogTag) {
+        if (key == null) return 0;
+        JSONArray cur = all(ctx);
+        int count = 0;
+        for (int i = 0; i < cur.length(); i++) {
+            JSONObject o = cur.optJSONObject(i);
+            if (o != null && key.equals(o.optString("key"))) {
+                count = (o.optLong("catTag", Long.MIN_VALUE) == catalogTag) ? o.optInt("attempts", 0) + 1 : 1;
+                try { o.put("attempts", count).put("catTag", catalogTag); } catch (Exception ignored) {}
+                break;
+            }
+        }
+        prefs(ctx).edit().putString(KEY, cur.toString()).apply();
+        return count;
+    }
 }