Generate partitions/subsets of knowledge split by ioos_category

.gitignore

@@ -33,3 +33,6 @@ cli/dump
 # find_standards_app
 utils/migrations
 utils/db.sqlite3
+
+# Generated partition artifacts (produced by utils/generate_partitions.py)
+data/

js/scripts/gen-data.mjs

@@ -6,9 +6,9 @@
  * Run automatically as part of `npm run wasm`.
  */
 
-import { mkdirSync, readdirSync, readFileSync, writeFileSync } from "fs";
-import { basename, dirname, join } from "path";
-import { fileURLToPath } from "url";
+import { mkdirSync, readdirSync, readFileSync, writeFileSync } from "node:fs";
+import { basename, dirname, join } from "node:path";
+import { fileURLToPath } from "node:url";
 import { parse as parseYaml } from "yaml";
 
 const __dirname = dirname(fileURLToPath(import.meta.url));
@@ -17,10 +17,23 @@ const publicDir = join(__dirname, "../public");
 
 mkdirSync(publicDir, { recursive: true });
 
-// Copy the CF vocabulary YAML as-is — the Rust YAML parser handles it.
-const cfYaml = readFileSync(join(standardsDir, "_cf_standards.yaml"));
-writeFileSync(join(publicDir, "cf_standards.yaml"), cfYaml);
+// Parse the CF vocabulary — both copy it for the Rust YAML path and parse it
+// for building JSON partition files.
+const cfYamlText = readFileSync(
+	join(standardsDir, "_cf_standards.yaml"),
+	"utf-8",
+);
+writeFileSync(join(publicDir, "cf_standards.yaml"), cfYamlText);
 console.log("gen-data: copied cf_standards.yaml");
+const cfRaw = parseYaml(cfYamlText);
+// YAML parses bare numbers (e.g. `unit: 1`) as JS numbers, but Rust's
+// CfStandard.{unit,description} are String fields. Coerce before JSON encoding.
+for (const std of Object.values(cfRaw.standard_names ?? {})) {
+	if (typeof std.unit === "number") std.unit = String(std.unit);
+	if (typeof std.description === "number")
+		std.description = String(std.description);
+}
+const cf = cfRaw;
 
 // Combine every per-standard YAML file into a single knowledge.json that
 // matches the Vec<Knowledge> JSON format expected by load_knowledge_from_json.
@@ -51,3 +64,67 @@ writeFileSync(join(publicDir, "knowledge.json"), JSON.stringify(knowledge));
 console.log(
 	`gen-data: combined ${knowledge.length} knowledge files → public/knowledge.json`,
 );
+
+// ── Partition generation ──────────────────────────────────────────────────────
+// Mirrors utils/generate_partitions.py but writes to public/data/ for Vite.
+
+function slugify(name) {
+	return name
+		.toLowerCase()
+		.replace(/[^a-z0-9]+/g, "_")
+		.replace(/^_|_$/g, "");
+}
+
+function subsetCf(cfData, nameSet) {
+	const standardNames = Object.fromEntries(
+		Object.entries(cfData.standard_names ?? {}).filter(([k]) => nameSet.has(k)),
+	);
+	const aliases = Object.fromEntries(
+		Object.entries(cfData.aliases ?? {}).filter(([, v]) => nameSet.has(v)),
+	);
+	return { standard_names: standardNames, aliases };
+}
+
+function writeJson(path, data) {
+	writeFileSync(path, JSON.stringify(data));
+}
+
+const dataDir = join(publicDir, "data");
+mkdirSync(join(dataDir, "partitions"), { recursive: true });
+
+// all-standards — full CF vocabulary as JSON
+writeJson(join(dataDir, "all-standards.json"), {
+	cf_standards: {
+		standard_names: cf.standard_names ?? {},
+		aliases: cf.aliases ?? {},
+	},
+});
+
+// Per-IOOS-category partitions
+const categories = {};
+for (const item of knowledge) {
+	if (item.ioos_category) {
+		if (!categories[item.ioos_category]) categories[item.ioos_category] = [];
+		categories[item.ioos_category].push(item);
+	}
+}
+
+for (const [category, items] of Object.entries(categories)) {
+	const slug = slugify(category);
+	const names = new Set(items.map((i) => i.name));
+	writeJson(join(dataDir, "partitions", `${slug}.json`), {
+		cf_standards: subsetCf(cf, names),
+		knowledge: items,
+	});
+}
+
+// all-knowledge — every standard with community knowledge
+const allNames = new Set(knowledge.map((i) => i.name));
+writeJson(join(dataDir, "all-knowledge.json"), {
+	cf_standards: subsetCf(cf, allNames),
+	knowledge,
+});
+
+console.log(
+	`gen-data: generated ${Object.keys(categories).length} category partitions → public/data/`,
+);

js/tests/api.test.ts

@@ -111,3 +111,56 @@
 		expect(categories).toContain("Wind");
 	});
 });
+
+describe("partition loading", () => {
+	test("meteorology partition is self-contained", async () => {
+		const resp = await fetch("/data/partitions/meteorology.json");
+		if (!resp.ok) throw new Error(`HTTP ${resp.status}`);
+		const { cf_standards, knowledge } = await resp.json();
+
+		const partLib = new StandardsLibrary();
+		partLib.loadStandards(cf_standards);
+		partLib.loadKnowledgeObjects(knowledge);
+		partLib.loadTestSuites();
+
+		// Meteorology standard resolves with enriched knowledge
+		const std = partLib.get("air_temperature");
+		expect(std.ioosCategory).toBe("Meteorology");
+		expect(std.commonVariableNames).toContain("temperature");
+
+		// Standard from a different category is absent from this partition
+		expect(() => partLib.get("sea_surface_temperature")).toThrow();
+	});
+
+	test("all-knowledge partition covers every knowledge entry", async () => {
+		const resp = await fetch("/data/all-knowledge.json");
+		if (!resp.ok) throw new Error(`HTTP ${resp.status}`);
+		const { cf_standards, knowledge } = await resp.json();
+
+		const partLib = new StandardsLibrary();
+		partLib.loadStandards(cf_standards);
+		partLib.loadKnowledgeObjects(knowledge);
+
+		// Both meteorology and temperature knowledge should be present
+		expect(partLib.get("air_temperature").ioosCategory).toBe("Meteorology");
+		expect(partLib.get("sea_surface_temperature").ioosCategory).toBe(
+			"Temperature",
+		);
+	});
+
+	test("all-standards partition loads the full CF vocabulary", async () => {
+		const resp = await fetch("/data/all-standards.json");
+		if (!resp.ok) throw new Error(`HTTP ${resp.status}`);
+		const { cf_standards } = await resp.json();
+
+		const partLib = new StandardsLibrary();
+		partLib.loadStandards(cf_standards);
+
+		// Any CF standard name should be resolvable (no knowledge enrichment)
+		const std = partLib.get("air_pressure_at_mean_sea_level");
+		expect(std.name).toBe("air_pressure_at_mean_sea_level");
+		expect(std.unit).toBe("Pa");
+		// ioosCategory is absent without knowledge (Option::None → undefined)
+		expect(std.ioosCategory).toBeUndefined();
+	});
+});

noxfile.py

@@ -195,6 +195,26 @@ def rust_update_tests(session: nox.Session) -> None:
     )
 
 
+@nox.session(python=False, default=False)
+def generate_partitions(session: nox.Session) -> None:
+    """Generate per-IOOS-category, all-knowledge, and all-standards partition
+    files (JSON + YAML) from core/standards/*.yaml → data/.
+
+    These are the canonical release artifacts consumed by Phase 2e CI uploads
+    and (eventually) the npm data package. The JS dev/test workflow generates
+    its own copy via ``npm run wasm`` → gen-data.mjs → public/data/.
+
+    Run: nox -s generate_partitions
+    """
+    session.run(
+        "uv",
+        "run",
+        "--script",
+        "utils/generate_partitions.py",
+        external=True,
+    )
+
+
 @nox.session(python=False)
 def js_test(session: nox.Session) -> None:
     """Run the JavaScript/WASM tests: Vitest API suite, Playwright demo E2E,

utils/generate_partitions.py

@@ -0,0 +1,151 @@
+#!/usr/bin/env -S uv run --script
+# /// script
+# requires-python = ">=3.12"
+# dependencies = ["pyyaml"]
+# ///
+"""
+Generate per-IOOS-category, all-knowledge, and all-standards partition
+files from core/standards/*.yaml.
+
+Writes compact JSON and human-readable YAML to data/:
+  data/all-standards.{json,yaml}          — full CF vocabulary
+  data/all-knowledge.{json,yaml}          — CF standards with knowledge + all knowledge
+  data/partitions/{category}.{json,yaml}  — per-IOOS-category subset (self-contained)
+
+Each partition file uses the format:
+  { "cf_standards": { "standard_names": {...}, "aliases": {...} },
+    "knowledge": [...] }
+
+JSON is compact (browser-decompressable via DecompressionStream).
+YAML is pretty-printed for human readability / diffing / sanity-checking.
+
+Consumers call library.loadStandards(data.cf_standards) then
+library.loadKnowledgeObjects(data.knowledge) to ingest a partition.
+
+Run: uv run --script utils/generate_partitions.py
+"""
+
+import json
+import re
+from pathlib import Path
+
+import yaml
+
+ROOT = Path(__file__).parent.parent
+STANDARDS_DIR = ROOT / "core" / "standards"
+DATA_DIR = ROOT / "data"
+
+
+def slugify(name: str) -> str:
+    """'Sea Level' → 'sea_level', 'Temperature' → 'temperature'"""
+    return re.sub(r"[^a-z0-9]+", "_", name.lower()).strip("_")
+
+
+def load_cf() -> dict:
+    with (STANDARDS_DIR / "_cf_standards.yaml").open() as f:
+        data = yaml.safe_load(f)
+    # YAML parses bare numbers (e.g. `unit: 1`) as Python ints/floats, but
+    # Rust's CfStandard.{unit,description} are String fields. Coerce so the
+    # JSON output is accepted by load_cf_standards_from_json.
+    for std in data.get("standard_names", {}).values():
+        if "unit" in std and not isinstance(std["unit"], str):
+            std["unit"] = str(std["unit"])
+        if "description" in std and not isinstance(std["description"], str):
+            std["description"] = str(std["description"])
+    return data
+
+
+def load_knowledge() -> list[dict]:
+    """Parse every per-standard YAML file into Knowledge-struct-compatible dicts."""
+    items = []
+    for path in sorted(STANDARDS_DIR.glob("*.yaml")):
+        if path.name.startswith("_"):
+            continue
+        name = path.stem
+        raw = yaml.safe_load(path.read_text(encoding="utf-8")) or {}
+        items.append(
+            {
+                "name": raw.get("name") or name,
+                "long_name": raw.get("long_name"),
+                "ioos_category": raw.get("ioos_category"),
+                "common_variable_names": raw.get("common_variable_names") or [],
+                "related_standards": raw.get("related_standards") or [],
+                "sibling_standards": raw.get("sibling_standards") or [],
+                "extra_attrs": raw.get("extra_attrs") or {},
+                "other_units": raw.get("other_units") or [],
+                # Normalise both singular ("comment") and plural ("comments") spellings
+                "comments": raw.get("comments") or raw.get("comment"),
+                "qc": raw.get("qc"),
+            }
+        )
+    return items
+
+
+def subset_cf(cf: dict, names: set[str]) -> dict:
+    """Return a CfYaml-format dict containing only the requested standard names."""
+    return {
+        "standard_names": {k: v for k, v in cf["standard_names"].items() if k in names},
+        "aliases": {k: v for k, v in cf.get("aliases", {}).items() if v in names},
+    }
+
+
+def emit(stem: Path, data: dict) -> None:
+    """Write compact JSON and pretty YAML for the same data."""
+    stem.parent.mkdir(parents=True, exist_ok=True)
+
+    json_path = stem.with_suffix(".json")
+    json_path.write_text(
+        json.dumps(data, separators=(",", ":")) + "\n", encoding="utf-8"
+    )
+
+    yaml_path = stem.with_suffix(".yaml")
+    yaml_path.write_text(
+        yaml.dump(data, allow_unicode=True, sort_keys=True), encoding="utf-8"
+    )
+
+    kb_json = json_path.stat().st_size // 1024
+    kb_yaml = yaml_path.stat().st_size // 1024
+    rel = stem.relative_to(ROOT)
+    print(f"  {rel}.json  ({kb_json} KB)  |  {rel}.yaml  ({kb_yaml} KB)")
+
+
+def main() -> None:
+    cf = load_cf()
+    knowledge = load_knowledge()
+
+    # 1. all-standards — the full CF vocabulary (no knowledge enrichment)
+    emit(
+        DATA_DIR / "all-standards",
+        {
+            "cf_standards": {
+                "standard_names": cf["standard_names"],
+                "aliases": cf.get("aliases", {}),
+            }
+        },
+    )
+
+    # 2. Per-IOOS-category partitions
+    categories: dict[str, list[dict]] = {}
+    for item in knowledge:
+        if cat := item.get("ioos_category"):
+            categories.setdefault(cat, []).append(item)
+
+    for category, items in sorted(categories.items()):
+        names = {item["name"] for item in items}
+        emit(
+            DATA_DIR / "partitions" / slugify(category),
+            {"cf_standards": subset_cf(cf, names), "knowledge": items},
+        )
+
+    # 3. all-knowledge — every standard that has community knowledge
+    all_names = {item["name"] for item in knowledge}
+    emit(
+        DATA_DIR / "all-knowledge",
+        {"cf_standards": subset_cf(cf, all_names), "knowledge": knowledge},
+    )
+
+    print(f"\n{len(categories)} category partitions → {DATA_DIR.relative_to(ROOT)}/")
+
+
+if __name__ == "__main__":
+    main()