From 44607ca896282e2df5881e73438b42879a3a41d4 Mon Sep 17 00:00:00 2001 From: Lann Martin Date: Sun, 9 Aug 2026 18:42:17 -0400 Subject: [PATCH] ct-runner: feature-tag scheduling from the suite's own inventory (#25) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The runner now reads polymorph-test's L0 tags inventory — newline- delimited 'name tag...' records in component-test:tags@0.1 custom sections, collected from the component and nested modules/components (ct-runner/src/tags.ts, ported from their inventory.rs; the SDK embeds records in the guest core module via #[link_section] and they survive wac composition, verified on the polymorph-tls composed suites). When an inventory exists, gating is on: harness.mjs runCases order (only, drift, applicability), N/A rows in the embed runner's exact wire shape (status/first-excluding-mark detail/diagnostics-complete:true), envelope scheduling 'tags', RunCounts.na, --missing f1,f2 on the CLI. Suites without an inventory run feature-blind as before; --missing without an inventory refuses rather than silently degrading (their runner's posture). Drift (an enumerated case no record covers) throws — unsound, not failing. Also the #26-review parity nits: returned rows now emit diagnostics-complete:true (trap/timeout already emitted false), and smoke-tls's dead webcryptoImports import is gone. smoke-tls: per-target missing-features lists; every TAG-GATING xfail went stale and is pruned — zero failures, zero xfails on all three compositions (delegated: decline N/A; plain: both delegated-signer cases N/A, decline runs and passes). The websocket suite carries an inventory with no tagged cases: gating activates with an empty schedule, 55/55 unchanged. Pin-bump note for polymorph-test's verify-deltic lane (not affected today, pinned pre-58875e8): at the next bump their fixture leg gains hsm N/A rows + envelope scheduling 'tags', and their runner.ts needs an N/A branch — regenerate the lane goldens then. Gates: ct-runner 17/17 (seeds unset/1/4242), runtime suite, websocket conformance 55/55, smoke-tls PASS. Closes #25. --- ct-runner/src/main.ts | 19 +- ct-runner/src/run-suite.ts | 77 ++++++- ct-runner/src/tags.ts | 173 ++++++++++++++++ ct-runner/tests/e2e_test.ts | 4 +- ct-runner/tests/expected/test-suite.jsonl | 12 +- ct-runner/tests/golden_test.ts | 2 +- ct-runner/tests/support.ts | 8 + ct-runner/tests/tags_test.ts | 240 ++++++++++++++++++++++ docs/consumers.md | 2 +- tools/smoke-tls/run.ts | 46 ++--- 10 files changed, 538 insertions(+), 45 deletions(-) create mode 100644 ct-runner/src/tags.ts create mode 100644 ct-runner/tests/tags_test.ts diff --git a/ct-runner/src/main.ts b/ct-runner/src/main.ts index 02471a9..44df70c 100644 --- a/ct-runner/src/main.ts +++ b/ct-runner/src/main.ts @@ -4,7 +4,8 @@ // deno run -A ct-runner/src/main.ts --out results.jsonl \ // [--translator ] [--imports ] \ // [--target NAME] [--suite-name NAME] \ -// [--only SUBSTRING] [--case-timeout-ms N] [--no-fresh-cases] [--jspi] +// [--only SUBSTRING] [--missing f1,f2,...] [--case-timeout-ms N] \ +// [--no-fresh-cases] [--jspi] // // `--imports ` convention (contracts/embedder-api.md §"Module // wiring and instantiation"): a TS module whose default export is either @@ -30,7 +31,8 @@ function usageError(msg: string): never { "usage: deno run -A ct-runner/src/main.ts --out " + "[--translator ] [--imports ] " + "[--target NAME] [--suite-name NAME] " + - "[--only SUBSTRING] [--case-timeout-ms N] [--no-fresh-cases] [--jspi]", + "[--only SUBSTRING] [--missing f1,f2,...] [--case-timeout-ms N] " + + "[--no-fresh-cases] [--jspi]", ); Deno.exit(2); } @@ -43,6 +45,7 @@ interface Cli { target: string; suiteName?: string; only?: string; + missing?: string[]; caseTimeoutMs?: number; freshCases: boolean; jspi: boolean; @@ -56,6 +59,7 @@ function parseArgs(argv: string[]): Cli { let target = "deltic/host"; let suiteName: string | undefined; let only: string | undefined; + let missing: string[] | undefined; let caseTimeoutMs: number | undefined; let freshCases = true; let jspi = false; @@ -81,6 +85,11 @@ function parseArgs(argv: string[]): Cli { case "--only": only = argv[++i]; break; + case "--missing": + // Comma-separated missing-feature list (upstream ct-runner's + // --missing f1,f2,...); tag gating per src/tags.ts. + missing = argv[++i].split(",").filter((f) => f !== ""); + break; case "--case-timeout-ms": caseTimeoutMs = Number(argv[++i]); break; @@ -105,6 +114,7 @@ function parseArgs(argv: string[]): Cli { target, suiteName, only, + missing, caseTimeoutMs, freshCases, jspi, @@ -192,6 +202,7 @@ async function main() { target: cli.target, suiteName: cli.suiteName ?? suiteNameFrom(cli.suitePath), only: cli.only, + missing: cli.missing, caseTimeoutMs: cli.caseTimeoutMs, freshCases: cli.freshCases, jspi: cli.jspi, @@ -200,8 +211,8 @@ async function main() { }); await Deno.writeTextFile(cli.out, lines.join("\n") + "\n"); console.error( - `${counts.passed} passed | ${counts.failed} failed | ${counts.skipped} skipped ` + - `(${counts.total} total) -> ${cli.out}`, + `${counts.passed} passed | ${counts.failed} failed | ${counts.skipped} skipped | ` + + `${counts.na} n/a (${counts.total} total) -> ${cli.out}`, ); if (counts.failed > 0) Deno.exit(1); } catch (e) { diff --git a/ct-runner/src/run-suite.ts b/ct-runner/src/run-suite.ts index d7e5150..4d9a2fe 100644 --- a/ct-runner/src/run-suite.ts +++ b/ct-runner/src/run-suite.ts @@ -19,6 +19,12 @@ import { } from "../../runtime/src/embedder/mod.ts"; import { Context, testContextImportRecord } from "./context.ts"; import { analyzeImports, requireImportsResolved } from "./import-analysis.ts"; +import { + applies, + firstExcluding, + loadTagsInventory, + tagsOf, +} from "./tags.ts"; /** The suite's `tests` interface id (wit/tests.wit `interface tests`, v0.1.0). */ export const TESTS_INTERFACE = "polymorph:test/tests@0.1.0"; @@ -44,6 +50,17 @@ export interface RunSuiteOptions { /** Substring filter: non-matching cases are skipped entirely (no emit), * per js/viewer/harness.mjs `runCases`'s `only` handling. */ only?: string; + /** + * Feature-tag scheduling (issue #25): the features this target LACKS — + * js/viewer/harness.mjs's `missing`. Tag-gating activates whenever the + * suite carries a `component-test:tags@0.1` inventory (src/tags.ts): + * non-applicable cases emit `not-applicable` rows instead of executing, + * and an enumerated case no record covers throws (inventory drift — the + * run is unsound, not failing). Passing `missing` for a suite WITHOUT an + * inventory is an error (gating requested but impossible — upstream's + * runner refuses the same way rather than silently degrading). + */ + missing?: string[]; /** * Per-case wall-clock budget in ms (the `--case-timeout` runner option * documented in harness.mjs's `runSuiteJsonl` doc comment). On expiry the @@ -76,6 +93,8 @@ export interface RunCounts { passed: number; failed: number; skipped: number; + /** Cases scheduled out as `not-applicable` (tag gating; harness.mjs `na`). */ + na: number; total: number; } @@ -143,18 +162,34 @@ export async function runSuite( const censusTests = await newTests(); const census = await censusTests.all(); + // Feature-tag scheduling (issue #25): gate on the suite's own + // `component-test:tags@0.1` inventory when it has one — the SDK embeds it + // in the guest core module and it survives wac composition, so this + // introspecting runner CAN see it (revising the earlier "cannot see the + // tags section" stance recorded below). Suites without an inventory run + // feature-blind exactly as before. + const inventory = loadTagsInventory(artifacts.componentBytes); + const missing = opts.missing ?? []; + if (inventory === null && opts.missing !== undefined) { + throw new Error( + "missing-features given, but the suite carries no " + + "component-test:tags@0.1 inventory (not built with their SDK, or " + + "sections stripped) — tag gating is impossible, refusing to " + + "silently run feature-blind", + ); + } + const suiteName = opts.suiteName.replaceAll("-", "_"); const artifactSha256 = await sha256Hex(artifacts.componentBytes); opts.emit(JSON.stringify({ "component-test-results": "0.1", target: opts.target, suite: { name: suiteName, "artifact-sha256": artifactSha256 }, - // "none": this runner applies no tag/manifest scheduling — it executes - // every enumerated case (component-test-results/src/lib.rs `RunInfo` - // doc comment: "none" is for producers that "cannot see the tags - // section", which is exactly this L1-direct runner's position — it never - // reads L0 static feature-tag metadata at all). - run: { segment: 0, scheduling: "none" }, + // "tags" when this run schedules against the suite's tag inventory, + // "none" for inventory-less suites (component-test-results/src/lib.rs + // `RunInfo`: "none" is for producers that cannot see the tags section + // — with the inventory in hand, this runner no longer is one). + run: { segment: 0, scheduling: inventory !== null ? "tags" : "none" }, })); if (census.length === 0) { @@ -166,7 +201,7 @@ export async function runSuite( ); } - const counts: RunCounts = { passed: 0, failed: 0, skipped: 0, total: 0 }; + const counts: RunCounts = { passed: 0, failed: 0, skipped: 0, na: 0, total: 0 }; for (const [i, testCase] of census.entries()) { const name = String(await testCase.name()); @@ -175,6 +210,28 @@ export async function runSuite( // continue" — a filtered-out case is skipped entirely, no emit. if (opts.only && !name.includes(opts.only)) continue; + // harness.mjs `runCases` mark scheduling, in its exact order: `only` + // first (above), then drift, then applicability. The N/A row's shape is + // the embed runner's (expected/verify-pipeline-fixture.jsonl): + // status, first excluding mark as detail, diagnostics-complete true. + if (inventory !== null) { + const tags = tagsOf(inventory, name); + if (tags === undefined) { + throw new Error(`inventory drift: no tags record covers ${name}`); + } + if (!applies(tags, missing)) { + counts.na++; + opts.emit(JSON.stringify({ + case: name, + status: "not-applicable", + detail: firstExcluding(tags, missing), + "diagnostics-complete": true, + })); + opts.log?.(`${name} … not-applicable`); + continue; + } + } + // js/viewer/harness.mjs `runCases`' `freshCases` branch: re-enumerate // from a fresh instance and run the matching case; a vanished case is // inventory drift, not a failing case, and throws. @@ -230,6 +287,10 @@ export async function runSuite( status: "pass", provenance: "returned", "duration-ms": durationMs, + // The case returned normally, so its diagnostics sideband is + // complete (upstream emits this on every returned row; deltic's + // trap/timeout rows already carry `false`). + "diagnostics-complete": true, }; } } catch (e) { @@ -247,6 +308,7 @@ export async function runSuite( provenance: "returned", detail: payload.val, "duration-ms": durationMs, + "diagnostics-complete": true, }; } else if (payload?.tag === "skipped") { counts.skipped++; @@ -256,6 +318,7 @@ export async function runSuite( provenance: "returned", detail: payload.val, "duration-ms": durationMs, + "diagnostics-complete": true, }; } else { // Contract violation: `outcome` has exactly two cases. Treat as diff --git a/ct-runner/src/tags.ts b/ct-runner/src/tags.ts new file mode 100644 index 0000000..b23e473 --- /dev/null +++ b/ct-runner/src/tags.ts @@ -0,0 +1,173 @@ +// Feature-tag scheduling (issue #25): the L0 tags inventory and the +// applicability rule, ported from polymorph-test's authorities — +// +// - Section format: crates/component-test-formats/src/inventory.rs +// (`collect_tags_sections` / `parse_tags_records`): newline-delimited +// `name tag...` text records in `component-test:tags@0.1` custom +// sections, collected from the component AND nested modules/components +// (their reader uses wasmparser's `parse_all`, which descends; the +// SDK's `#[link_section]` puts the records in the guest CORE module, +// and those survive wac composition — verified empirically on the +// polymorph-tls composed suites). Records are newline-delimited within +// a section; a producer may omit the final newline, so a newline is +// repaired per section before concatenation, exactly as upstream does. +// - Record forms: `name tag...` (exact) and `prefix/* tag...` (generated +// rows: leaves are enumerated at run time below the prefix). +// - Applicability: crates/component-test-core/src/tags.rs — `feature` +// requires the target to HAVE the feature, `!feature` requires it to +// LACK it; a case applies iff every mark is satisfied against the +// runner's missing-features list. js/viewer/harness.mjs `applies()` is +// the JS-leg reference this mirrors. +// - Drift policy: harness.mjs `runCases` — an enumerated case that no +// record covers throws (the run is unsound, not failing). + +/** The custom-section name (component-test-core `name::TAGS_SECTION`). */ +export const TAGS_SECTION = "component-test:tags@0.1"; + +/** Parsed static inventory: exact case records + generated-row prefixes. */ +export interface TagsInventory { + exact: Map; + prefixes: Array<{ prefix: string; tags: string[] }>; +} + +const MAGIC = [0x00, 0x61, 0x73, 0x6d]; // "\0asm" + +function hasWasmMagic(bytes: Uint8Array, at = 0): boolean { + return bytes.length >= at + 8 && MAGIC.every((b, i) => bytes[at + i] === b); +} + +/** u32 LEB128 at `pos`; returns [value, nextPos]. Traps on overlong/EOF. */ +function lebU32(bytes: Uint8Array, pos: number): [number, number] { + let result = 0; + let shift = 0; + for (;;) { + if (pos >= bytes.length) throw new Error("tags scan: truncated LEB128"); + const b = bytes[pos++]; + result |= (b & 0x7f) << shift; + if ((b & 0x80) === 0) break; + shift += 7; + if (shift >= 35) throw new Error("tags scan: LEB128 too long for u32"); + } + return [result >>> 0, pos]; +} + +/** + * Collect the concatenated bytes of every `component-test:tags@0.1` custom + * section in `bytes` — the component's own sections plus those of nested + * core modules (section id 1) and nested components (section id 4), which + * both embed complete wasm binaries. Returns null when no section exists + * anywhere (a suite not built with their SDK). + */ +export function collectTagsSections(bytes: Uint8Array): Uint8Array | null { + if (!hasWasmMagic(bytes)) throw new Error("tags scan: not a wasm binary"); + const chunks: Uint8Array[] = []; + const decoder = new TextDecoder(); + const NL = new Uint8Array([0x0a]); + + const scan = (buf: Uint8Array, core: boolean) => { + let pos = 8; // magic + version/layer + while (pos < buf.length) { + const id = buf[pos++]; + const [size, afterSize] = lebU32(buf, pos); + pos = afterSize; + const end = pos + size; + if (end > buf.length) throw new Error("tags scan: truncated section"); + if (id === 0) { + const [nameLen, afterName] = lebU32(buf, pos); + const nameEnd = afterName + nameLen; + if (nameEnd > end) throw new Error("tags scan: truncated custom name"); + if (decoder.decode(buf.subarray(afterName, nameEnd)) === TAGS_SECTION) { + const data = buf.subarray(nameEnd, end); + chunks.push(data); + // Newline repair per section (inventory.rs: nothing guarantees a + // producer terminates its last record). + if (data.length === 0 || data[data.length - 1] !== 0x0a) { + chunks.push(NL); + } + } + } else if (!core && (id === 1 || id === 4)) { + // 1 = core module, 4 = nested component: payload is a full binary. + const payload = buf.subarray(pos, end); + if (hasWasmMagic(payload)) scan(payload, id === 1); + } + pos = end; + } + }; + + scan(bytes, false); + if (chunks.length === 0) return null; + const total = chunks.reduce((n, c) => n + c.length, 0); + const out = new Uint8Array(total); + let off = 0; + for (const c of chunks) { + out.set(c, off); + off += c.length; + } + return out; +} + +/** + * Parse concatenated records (inventory.rs `parse_tags_records`): one + * record per line, `name tag...`, blank lines skipped, duplicate names and + * empty tags rejected. Grammar validation beyond that (WIT-label checks) + * is the producer's job — their SDK validates at macro-expansion time. + */ +export function parseTagsRecords(bytes: Uint8Array): TagsInventory { + const text = new TextDecoder("utf-8", { fatal: true }).decode(bytes); + const inv: TagsInventory = { exact: new Map(), prefixes: [] }; + const seen = new Set(); + for (const line of text.split("\n")) { + if (line.trim() === "") continue; + const parts = line.split(" ").filter((p) => p !== ""); + const name = parts[0]; + const tags = parts.slice(1); + for (const t of tags) { + if (t === "" || t === "!") { + throw new Error(`tags section: empty mark on \`${name}\``); + } + } + if (seen.has(name)) { + throw new Error(`tags section: duplicate record \`${name}\``); + } + seen.add(name); + const prefix = name.endsWith("/*") ? name.slice(0, -2) : null; + if (prefix !== null) { + inv.prefixes.push({ prefix, tags }); + } else { + inv.exact.set(name, tags); + } + } + return inv; +} + +/** Convenience: scan + parse; null when the suite carries no inventory. */ +export function loadTagsInventory(bytes: Uint8Array): TagsInventory | null { + const sections = collectTagsSections(bytes); + return sections === null ? null : parseTagsRecords(sections); +} + +/** The tags covering `name`: exact record, else a generated-row prefix + * record (leaves live below `prefix/`), else undefined (inventory drift). */ +export function tagsOf(inv: TagsInventory, name: string): string[] | undefined { + const exact = inv.exact.get(name); + if (exact !== undefined) return exact; + for (const { prefix, tags } of inv.prefixes) { + if (name.startsWith(prefix + "/")) return tags; + } + return undefined; +} + +/** harness.mjs `applies()`: `!f` needs f missing; `f` needs f present. */ +export function applies(tags: string[], missing: string[]): boolean { + return tags.every((t) => + t.startsWith("!") ? missing.includes(t.slice(1)) : !missing.includes(t) + ); +} + +/** The N/A row's `detail`: the first unsatisfied mark (harness.mjs's + * `excluding`), empty string if somehow none (mirrors `excluding ?? ""`). */ +export function firstExcluding(tags: string[], missing: string[]): string { + return tags.find((t) => + t.startsWith("!") ? !missing.includes(t.slice(1)) : missing.includes(t) + ) ?? ""; +} diff --git a/ct-runner/tests/e2e_test.ts b/ct-runner/tests/e2e_test.ts index b511f35..e96f972 100644 --- a/ct-runner/tests/e2e_test.ts +++ b/ct-runner/tests/e2e_test.ts @@ -20,7 +20,7 @@ Deno.test({ suiteName: "test-suite", emit: (l) => lines.push(l), }); - assertEq(counts, { passed: 4, failed: 1, skipped: 1, total: 6 }); + assertEq(counts, { passed: 4, failed: 1, skipped: 1, na: 0, total: 6 }); assertEq(lines.length, 1 + 6 + 1); // envelope + 6 cases + terminator const events = lines.slice(1, -1).map((l) => JSON.parse(l)); @@ -94,6 +94,6 @@ Deno.test({ freshCases: false, emit: (l) => lines.push(l), }); - assertEq(counts, { passed: 4, failed: 1, skipped: 1, total: 6 }); + assertEq(counts, { passed: 4, failed: 1, skipped: 1, na: 0, total: 6 }); }, }); diff --git a/ct-runner/tests/expected/test-suite.jsonl b/ct-runner/tests/expected/test-suite.jsonl index d0d0089..ff3d0c7 100644 --- a/ct-runner/tests/expected/test-suite.jsonl +++ b/ct-runner/tests/expected/test-suite.jsonl @@ -1,8 +1,8 @@ {"component-test-results":"0.1","target":"wasmtime/deltic","suite":{"name":"test_suite","artifact-sha256":""},"run":{"segment":0,"scheduling":"none"}} -{"case":"suite/basic/pass","status":"pass","provenance":"returned"} -{"case":"suite/basic/fail","status":"fail","provenance":"returned","detail":"expected 2 + 2 = 4, got 5"} -{"case":"suite/basic/skip","status":"skipped","provenance":"returned","detail":"declared hardware token unavailable at run time"} -{"case":"suite/diag/chatty","status":"pass","provenance":"returned","diagnostics":["starting the chatty case","midpoint observation: ok","finishing up"]} -{"case":"suite/diag/slow","status":"pass","provenance":"returned","diagnostics":["sleeping briefly"]} -{"case":"suite/nested/deep/leaf","status":"pass","provenance":"returned"} +{"case":"suite/basic/pass","status":"pass","provenance":"returned","diagnostics-complete":true} +{"case":"suite/basic/fail","status":"fail","provenance":"returned","detail":"expected 2 + 2 = 4, got 5","diagnostics-complete":true} +{"case":"suite/basic/skip","status":"skipped","provenance":"returned","detail":"declared hardware token unavailable at run time","diagnostics-complete":true} +{"case":"suite/diag/chatty","status":"pass","provenance":"returned","diagnostics-complete":true,"diagnostics":["starting the chatty case","midpoint observation: ok","finishing up"]} +{"case":"suite/diag/slow","status":"pass","provenance":"returned","diagnostics-complete":true,"diagnostics":["sleeping briefly"]} +{"case":"suite/nested/deep/leaf","status":"pass","provenance":"returned","diagnostics-complete":true} {"segment-end":true} diff --git a/ct-runner/tests/golden_test.ts b/ct-runner/tests/golden_test.ts index 1333bd5..3a88ce6 100644 --- a/ct-runner/tests/golden_test.ts +++ b/ct-runner/tests/golden_test.ts @@ -33,7 +33,7 @@ Deno.test({ suiteName: "test-suite", emit: (l) => lines.push(l), }); - assertEq(counts, { passed: 4, failed: 1, skipped: 1, total: 6 }); + assertEq(counts, { passed: 4, failed: 1, skipped: 1, na: 0, total: 6 }); const got = lines.map(normalize).join("\n") + "\n"; const want = await Deno.readTextFile( diff --git a/ct-runner/tests/support.ts b/ct-runner/tests/support.ts index e01c50b..fa0adae 100644 --- a/ct-runner/tests/support.ts +++ b/ct-runner/tests/support.ts @@ -25,6 +25,14 @@ export async function haveFixture(rel: string): Promise { export async function artifactsOf(rel: string): Promise { const componentBytes = (await readArtifact(rel))!; + return artifactsOfBytes(componentBytes); +} + +/** Translate caller-supplied component bytes (e.g. a fixture with a + * synthesized `component-test:tags@0.1` section appended). */ +export function artifactsOfBytes( + componentBytes: Uint8Array, +): ComponentArtifacts { const { plan, adapters } = translator!.translate(componentBytes); return { plan, componentBytes, adapters }; } diff --git a/ct-runner/tests/tags_test.ts b/ct-runner/tests/tags_test.ts new file mode 100644 index 0000000..ba2b17c --- /dev/null +++ b/ct-runner/tests/tags_test.ts @@ -0,0 +1,240 @@ +// Feature-tag scheduling (issue #25): the tags inventory parser/scanner and +// the gated case loop, against polymorph-test's authorities — +// crates/component-test-formats/src/inventory.rs (section format), +// crates/component-test-core/src/tags.rs (applicability), and +// js/viewer/harness.mjs `runCases` (scheduling order, N/A rows, drift). +// The N/A wire shape is pinned to the embed runner's golden +// (expected/verify-pipeline-fixture.jsonl): +// {"case":…,"status":"not-applicable","detail":"", +// "diagnostics-complete":true} + +import { assertEq } from "../../runtime/tests/support/asserts.ts"; +import { + applies, + collectTagsSections, + firstExcluding, + loadTagsInventory, + parseTagsRecords, + TAGS_SECTION, + tagsOf, +} from "../src/tags.ts"; +import { runSuite } from "../src/mod.ts"; +import { + artifactsOfBytes, + haveFixture, + readArtifact, + TEST_SUITE_WASM, +} from "./support.ts"; + +const ready = await haveFixture(TEST_SUITE_WASM); +const enc = new TextEncoder(); + +function leb(n: number): number[] { + const out: number[] = []; + do { + let b = n & 0x7f; + n >>>= 7; + if (n !== 0) b |= 0x80; + out.push(b); + } while (n !== 0); + return out; +} + +/** Encode one custom section frame (id 0, name, data). */ +function customSection(name: string, data: Uint8Array): Uint8Array { + const nameBytes = enc.encode(name); + const payload = [...leb(nameBytes.length), ...nameBytes, ...data]; + return new Uint8Array([0x00, ...leb(payload.length), ...payload]); +} + +/** Append a `component-test:tags@0.1` section to a wasm binary (custom + * sections are legal anywhere after the preamble; appending is simplest). */ +function withTags(bytes: Uint8Array, records: string): Uint8Array { + const section = customSection(TAGS_SECTION, enc.encode(records)); + const out = new Uint8Array(bytes.length + section.length); + out.set(bytes, 0); + out.set(section, bytes.length); + return out; +} + +// --- parser ------------------------------------------------------------------ + +Deno.test("tags: records parse (exact + generated prefix), lookup + applies", () => { + const inv = parseTagsRecords(enc.encode( + "a/b/pass\n" + + "a/b/gated hsm\n" + + "a/b/decline !hsm\n" + + "\n" + // blank lines skipped + "a/gen/* slow net\n", + )); + assertEq(tagsOf(inv, "a/b/pass"), []); + assertEq(tagsOf(inv, "a/b/gated"), ["hsm"]); + assertEq(tagsOf(inv, "a/gen/tc1"), ["slow", "net"]); // prefix record + assertEq(tagsOf(inv, "a/gen"), undefined); // prefix needs a leaf below it + assertEq(tagsOf(inv, "a/b/unknown"), undefined); // drift, caller's problem + + // component-test-core/src/tags.rs applicability: `f` needs f present, + // `!f` needs f missing; unmarked applies everywhere. + assertEq(applies([], ["hsm"]), true); + assertEq(applies(["hsm"], []), true); + assertEq(applies(["hsm"], ["hsm"]), false); + assertEq(applies(["!hsm"], []), false); + assertEq(applies(["!hsm"], ["hsm"]), true); + assertEq(applies(["slow", "net"], ["net"]), false); + + // harness.mjs `excluding`: the FIRST unsatisfied mark, as the N/A detail. + assertEq(firstExcluding(["hsm"], ["hsm"]), "hsm"); + assertEq(firstExcluding(["!hsm"], []), "!hsm"); + assertEq(firstExcluding(["slow", "net"], ["net"]), "net"); +}); + +Deno.test("tags: duplicate records and empty marks are rejected (inventory.rs)", () => { + let threw = ""; + try { + parseTagsRecords(enc.encode("a/b\na/b\n")); + } catch (e) { + threw = String(e); + } + assertEq(threw.includes("duplicate record"), true); + try { + parseTagsRecords(enc.encode("a/b !\n")); + } catch (e) { + threw = String(e); + } + assertEq(threw.includes("empty mark"), true); +}); + +// --- section scanner --------------------------------------------------------- + +Deno.test("tags: scanner finds nested core-module sections and repairs newlines", () => { + // A minimal core module carrying a tags section WITHOUT a trailing + // newline, nested in a minimal component that carries a second section — + // the real layout (#[link_section] puts records in the guest core + // module) plus the concatenation/newline-repair path (inventory.rs). + const coreCustom = customSection(TAGS_SECTION, enc.encode("m/core hsm")); + const core = new Uint8Array([ + 0x00, 0x61, 0x73, 0x6d, 0x01, 0x00, 0x00, 0x00, // core preamble + ...coreCustom, + ]); + const moduleSection = new Uint8Array([0x01, ...leb(core.length), ...core]); + const componentCustom = customSection(TAGS_SECTION, enc.encode("m/comp\n")); + const component = new Uint8Array([ + 0x00, 0x61, 0x73, 0x6d, 0x0d, 0x00, 0x01, 0x00, // component preamble + ...moduleSection, + ...componentCustom, + ]); + + const collected = collectTagsSections(component); + assertEq(new TextDecoder().decode(collected!), "m/core hsm\nm/comp\n"); + const inv = parseTagsRecords(collected!); + assertEq(tagsOf(inv, "m/core"), ["hsm"]); + assertEq(tagsOf(inv, "m/comp"), []); + + // No section anywhere -> null (suite not built with their SDK). + const bare = new Uint8Array([0x00, 0x61, 0x73, 0x6d, 0x0d, 0x00, 0x01, 0x00]); + assertEq(collectTagsSections(bare), null); +}); + +// --- gated runs (e2e over the example suite + synthesized inventory) ---------- + +const RECORDS = "suite/basic/pass\n" + + "suite/basic/fail\n" + + "suite/basic/skip !hw\n" + + "suite/diag/chatty\n" + + "suite/diag/slow hw\n" + + "suite/nested/deep/leaf\n"; + +Deno.test({ + name: "tags e2e: missing feature schedules the requiring case out (N/A row exact)", + ignore: !ready, + fn: async () => { + const bytes = withTags((await readArtifact(TEST_SUITE_WASM))!, RECORDS); + const lines: string[] = []; + const counts = await runSuite(artifactsOfBytes(bytes), { + target: "deltic/test", + suiteName: "test-suite", + missing: ["hw"], + emit: (l) => lines.push(l), + }); + // hw missing: diag/slow (hw) is N/A; basic/skip (!hw) APPLIES and runs + // to its usual skipped verdict. + assertEq(counts, { passed: 3, failed: 1, skipped: 1, na: 1, total: 6 }); + + const envelope = JSON.parse(lines[0]); + assertEq(envelope.run.scheduling, "tags"); + + const rows = lines.slice(1, -1).map((l) => JSON.parse(l)); + const na = rows.find((r) => r.status === "not-applicable"); + // The embed runner's exact N/A shape (verify-pipeline-fixture.jsonl). + assertEq(na, { + case: "suite/diag/slow", + status: "not-applicable", + detail: "hw", + "diagnostics-complete": true, + }); + // Suite order preserved: N/A rows are emitted in place, not batched. + assertEq(rows.map((r) => r.case).indexOf("suite/diag/slow"), 4); + }, +}); + +Deno.test({ + name: "tags e2e: gating is on whenever an inventory exists (decline case N/As)", + ignore: !ready, + fn: async () => { + const bytes = withTags((await readArtifact(TEST_SUITE_WASM))!, RECORDS); + const lines: string[] = []; + const counts = await runSuite(artifactsOfBytes(bytes), { + target: "deltic/test", + suiteName: "test-suite", + // no `missing`: the !hw decline case does not apply on a + // fully-featured target (tags.rs polarity). + emit: (l) => lines.push(l), + }); + assertEq(counts, { passed: 4, failed: 1, skipped: 0, na: 1, total: 6 }); + const na = lines.slice(1, -1).map((l) => JSON.parse(l)) + .find((r) => r.status === "not-applicable"); + assertEq(na?.case, "suite/basic/skip"); + assertEq(na?.detail, "!hw"); + }, +}); + +Deno.test({ + name: "tags e2e: inventory drift (uncovered case) is unsound, throws", + ignore: !ready, + fn: async () => { + const partial = RECORDS.replace("suite/basic/pass\n", ""); + const bytes = withTags((await readArtifact(TEST_SUITE_WASM))!, partial); + let threw = ""; + try { + await runSuite(artifactsOfBytes(bytes), { + target: "deltic/test", + suiteName: "test-suite", + emit: () => {}, + }); + } catch (e) { + threw = String(e); + } + assertEq(threw.includes("inventory drift"), true); + assertEq(threw.includes("suite/basic/pass"), true); + }, +}); + +Deno.test({ + name: "tags e2e: --missing without an inventory refuses (no silent feature-blind run)", + ignore: !ready, + fn: async () => { + const bytes = (await readArtifact(TEST_SUITE_WASM))!; // no section + let threw = ""; + try { + await runSuite(artifactsOfBytes(bytes), { + target: "deltic/test", + suiteName: "test-suite", + missing: ["hw"], + emit: () => {}, + }); + } catch (e) { + threw = String(e); + } + assertEq(threw.includes("no component-test:tags@0.1 inventory"), true); + }, +}); diff --git a/docs/consumers.md b/docs/consumers.md index ea33354..4c9cf73 100644 --- a/docs/consumers.md +++ b/docs/consumers.md @@ -94,7 +94,7 @@ Reference implementations developed here, pending upstreaming | `exams/iroh-endpoint` | the endpoint exit exam | 5/5: bind+identity, relay echo, WebRTC upgrade, jco#11/#13 assertions, teardown | | `ct-runner` | L3 runner for the polymorph-test L1 contract | golden-tested L4 JSONL; drives the websocket suite | | `tools/smoke-c0` | C0 smoke legs + report | legs 1–4 (`REPORT.md`) | -| `tools/smoke-tls` | polymorph-tls conformance under deltic ([#18](https://github.com/lann/deltic/issues/18)) | translate 8/8; suites: all applicable cases green on every composition (sole named xfail class: tag-gating [#25](https://github.com/lann/deltic/issues/25); the callback-null-context defect it found, [#24](https://github.com/lann/deltic/issues/24), is fixed — attribution sentinels, `runtime/src/jspi/bridge.ts`) | +| `tools/smoke-tls` | polymorph-tls conformance under deltic ([#18](https://github.com/lann/deltic/issues/18)) | translate 8/8; suites: zero failures, zero xfails on every composition — tag gating ([#25](https://github.com/lann/deltic/issues/25), `ct-runner/src/tags.ts`) schedules the per-target inapplicable cases to `not-applicable` exactly like their harness legs; the callback-null-context defect it found ([#24](https://github.com/lann/deltic/issues/24)) is fixed — attribution sentinels, `runtime/src/jspi/bridge.ts` | Deferred consumer surfaces: experiment-mosh deep E2E ([#2](https://github.com/lann/deltic/issues/2)), webcrypto family completion diff --git a/tools/smoke-tls/run.ts b/tools/smoke-tls/run.ts index 136aba8..afd7c32 100644 --- a/tools/smoke-tls/run.ts +++ b/tools/smoke-tls/run.ts @@ -13,11 +13,12 @@ // READ-ONLY; nothing here writes to the polymorph trees. // // Named residues (conformance discipline: no unnamed absorption): -// TAG-GATING (#25) — their harness marks cases N/A per target via the L0 -// tags section (`missing`/`tagsOf`, run-node.mjs); ct-runner has no tag -// gating yet. Affected: `delegated/decline` (tagged !delegated-signer) -// fails on delegated compositions, and the plain composition fails its -// delegated-* cases. +// TAG-GATING (#25) — FIXED (ct-runner reads the suites' own +// `component-test:tags@0.1` inventory, ct-runner/src/tags.ts; the +// sections survive wac composition, verified on these artifacts). Each +// target below declares its missing-features and the previously +// xfailed cases schedule out as `not-applicable`, exactly like their +// harness legs; the xfail entries were pruned. // CALLBACK-NULL-CONTEXT (#24) — FIXED (continuation-chunk attribution // sentinels, jspi/bridge.ts); the entry below was pruned. The // webcrypto-composed target is the only corpus that reaches the @@ -36,7 +37,6 @@ import { import type { ComponentArtifacts } from "../../runtime/src/embedder/mod.ts"; import { runSuite } from "../../ct-runner/src/mod.ts"; import { wasiShims } from "../../wasi-shims/src/mod.ts"; -import { webcryptoImports } from "../../ports/webcrypto/src/mod.ts"; const CONF = `${POLYMORPH}/polymorph-tls/target/conformance`; @@ -58,22 +58,19 @@ const TRANSLATE_TARGETS: Array<[string, string]> = [ ], ]; -/** The executable smoke matrix: [target-key, artifact, xfails]. - * All compositions are self-contained (surfaces are pure WASI — phase 1), - * so no extra host modules are wired. Each xfail names its class + issue. */ -const EXEC_TARGETS: Array<[string, string, Record]> = [ - ["deltic-delegated", `${CONF}/suite-delegated.wasm`, { - "delegated/decline": "TAG-GATING #25 (!delegated-signer, N/A here)", - }], - ["deltic-delegated-webcrypto", `${CONF}/suite-delegated-webcrypto.wasm`, { - "delegated/decline": "TAG-GATING #25 (!delegated-signer, N/A here)", - }], - // Plain composition: `delegated/decline` PASSES here (it asserts exactly - // the no-signer refusal) and coexist-in-guest-ed25519 needs no signer; - // only `delegated/handshake` is genuinely signer-gated. - ["deltic-plain", `${CONF}/suite-plain.wasm`, { - "delegated/handshake": "TAG-GATING #25 (delegated-signer, N/A on plain)", - }], +/** The executable smoke matrix: [target-key, artifact, missing-features, + * xfails]. All compositions are self-contained (surfaces are pure WASI — + * phase 1), so no extra host modules are wired. `missing` mirrors what + * their harness legs pass per target (run-node.mjs); tag gating turns the + * per-target inapplicable cases into `not-applicable` rows. Any future + * xfail must name its class + issue. */ +const EXEC_TARGETS: Array<[string, string, string[], Record]> = [ + ["deltic-delegated", `${CONF}/suite-delegated.wasm`, [], {}], + ["deltic-delegated-webcrypto", `${CONF}/suite-delegated-webcrypto.wasm`, [], {}], + // Plain composition: no signer is wired, so the signer-gated case + // schedules out; `delegated/decline` (!delegated-signer) APPLIES here and + // passes (it asserts exactly the no-signer refusal). + ["deltic-plain", `${CONF}/suite-plain.wasm`, ["delegated-signer"], {}], ]; const CASE_TIMEOUT_MS = 60_000; // run-node.mjs's per-case wall bound. @@ -122,7 +119,7 @@ async function execPhase(only?: string): Promise { console.log("\n=== phase 2: execute the suites (ct-runner + wasi-shims) ===\n"); const t = await loadTranslator(); let failures = 0; - for (const [target, path, xfails] of EXEC_TARGETS) { + for (const [target, path, missing, xfails] of EXEC_TARGETS) { console.log(`--- ${target}: ${path}`); let componentBytes: Uint8Array; try { @@ -139,6 +136,7 @@ async function execPhase(only?: string): Promise { imports: wasiShims(), target, suiteName: path.split("/").pop()!.replace(/\.wasm$/, ""), + missing, only, caseTimeoutMs: CASE_TIMEOUT_MS, emit: (line) => lines.push(line), @@ -146,7 +144,7 @@ async function execPhase(only?: string): Promise { }); console.log( ` ${counts.passed} passed | ${counts.failed} failed | ` + - `${counts.skipped} skipped (${counts.total} total)`, + `${counts.skipped} skipped | ${counts.na} n/a (${counts.total} total)`, ); // deltic-plain: delegated-* failures are the KNOWN tag-gating delta // (header comment); anything else counts.