Implement Unicode 17 UAX #9 bidi analysis

.github/workflows/tests.yaml

@@ -52,3 +52,11 @@ jobs:
 
       - name: Run all tests
         run: ROC=roc ./scripts/all_tests.sh
+
+      - name: Upload minimized Bidi regressions
+        if: failure()
+        uses: actions/upload-artifact@v4
+        with:
+          name: bidi-regressions-${{ matrix.os }}
+          path: .roc-unicode-tmp/failures
+          if-no-files-found: ignore

README.md

@@ -140,6 +140,39 @@ core. Its package policy is independently versioned as
 `LineBreak.preserve_graphemes_revision`, and `LineBreak.profile_revision`
 distinguishes that policy axis from the Unicode/UAX version.
 
+## Bidirectional text
+
+`Bidi` implements Unicode 17.0.0 UAX #9 revision 51 at conformance level
+UAX9-C1: P1--P3, X1--X10, W1--W7, N0--N2, I1--I2, and the line-specific
+L1--L4 steps. `Bidi.analyze_paragraph` retains paragraph analysis facts rather
+than replacing logical text: original scalar/text ranges, resolved levels,
+logical runs, X9-removal state, paired brackets, and mirroring information.
+`Bidi.paragraph_ranges` follows P1, assigning paragraph separators to their
+preceding range and treating CRLF as one separator. Empty input returns one
+empty paragraph range; a final separator does not add an artificial empty
+paragraph.
+
+For a paragraph selected from a larger `Str`, use
+`Bidi.analyze_range(source, paragraph_range, direction, limits)`. It validates
+that the supplied `TextRange` is one of the P1 ranges and retains absolute byte
+and scalar coordinates from the full source; line ranges, visual-to-logical
+mappings, and visual runs use those same coordinates. This range API validates
+through the selected P1 boundary and replays the selected paragraph for
+analysis; `Bidi.analyze_paragraph` decodes its single paragraph once.
+`Bidi.reorder_line`
+then applies L1/L2 to an actual, paragraph-contained logical line. It does not
+shape Arabic or replace source scalars: L4 is returned as a mirrored-glyph
+requirement and optional best-fit mapping for renderers. Paragraph limits are
+checked before retained analysis is committed, with typed errors identifying
+the ingestion stage and source range.
+
+For a requested line whose scalar range starts at `line_start`,
+`visual_to_logical[visual_position]` is an absolute source scalar index, while
+`logical_to_visual[absolute_scalar_index - line_start]` is the matching visual
+position (or absent for X9-removed controls). This intentionally makes the
+forward mapping directly usable against retained source coordinates while the
+inverse stays compact for the requested line.
+
 ## Scripts and shaping-oriented itemization
 
 `Script` exposes Unicode 17's normative `Script` and `Script_Extensions`

benchmarks/bidi/README.md

@@ -0,0 +1,34 @@
+# Bidi retained-analysis benchmark
+
+`run.py` measures complete paragraph analysis plus one full and 64 partitioned
+logical-line L1/L2 reorders for all-LTR, mixed-script, neutral/NSM, valid
+nested isolates, overflowed controls, repeated FSI, 63/64-depth paired
+brackets, and many-run corpora. Each invocation reports a nonzero checksum and
+input byte count, so an optimized build cannot discard retained analysis or any
+line permutation. It verifies the configured compiler against `.roc-version`
+before building.
+
+```sh
+python3 benchmarks/bidi/run.py --roc "$ROC"
+```
+
+The generated JSON is intentionally untracked. Its `scaling_seconds_ratio`
+compares every adversarial corpus at the default 4 KiB, 16 KiB, and 64 KiB
+sizes. Retained state and work should grow linearly with paragraph size.
+
+## Optional external differential
+
+Normal CI uses the vendored Unicode 17 BidiTest and BidiCharacterTest data.
+`scripts/bidi_reference_differential.py` is an explicit local verification tool.
+It downloads Unicode's official Code9 C reference source for Unicode 17.0.0,
+verifies every source file against recorded SHA-256 digests, compiles an
+isolated query wrapper, and compares deterministic seeded scalar cases against
+the Roc test app. It verifies the Roc compiler against `.roc-version`; a failed
+download, checksum, compile, or mismatch fails the command. The source is not
+vendored, so the repository's canonical UCD data remains its local source of
+truth.
+
+On a differential failure, the command writes a minimized tab-separated row
+under `.roc-unicode-tmp/failures`. Re-run it by placing that row after a
+`ROC_UNICODE_TEST_V1` header for its documented suite, or promote the row to the
+adjacent Bidi test fixture after review.

benchmarks/bidi/main.roc

@@ -0,0 +1,53 @@
+app [run!] {
+	pf: platform "../../tests/platform/main.roc",
+	unicode: "../../package/main.roc",
+}
+
+import unicode.Bidi
+import unicode.ScalarRange
+import unicode.TextRange
+
+## Read `REPEATS\tLINE_REORDERS\nTEXT`, retaining a complete paragraph on
+## every repeat. The checksum makes both a whole-line reorder and a partition
+## into many logical line ranges observable.
+run! : Str => Str
+run! = |input| {
+	match input.split_on("\n") {
+		[repeat_text, source] => {
+			match repeat_text.split_on("\t") {
+				[repeats_text, line_reorders_text] => {
+					repeats = U64.from_str(repeats_text) ?? return "FAIL\tinvalid repeat count"
+					line_reorders = U64.from_str(line_reorders_text) ?? return "FAIL\tinvalid line reorder count"
+					if line_reorders < 1 {
+						"FAIL\tline reorder count must be positive"
+					} else {
+						var checksum = 0.U64
+						var at = 0.U64
+						while at < repeats {
+							analysis = Bidi.analyze_paragraph(source, Auto, Bidi.default_limits) ?? return "FAIL\tanalysis"
+							paragraph = TextRange.scalar_range(Bidi.paragraph_range(analysis))
+							whole_line = Bidi.reorder_line(analysis, paragraph) ?? return "FAIL\twhole line"
+							start = ScalarRange.start(paragraph)
+							end = ScalarRange.end(paragraph)
+							width = end - start
+							var line_index = 0.U64
+							checksum = checksum + Bidi.entries(analysis).len() + Bidi.visual_to_logical(whole_line).len()
+							while line_index < line_reorders {
+								line_start = start + width * line_index / line_reorders
+								line_end = start + width * (line_index + 1) / line_reorders
+								line_range = ScalarRange.from_bounds(line_start, line_end) ?? return "FAIL\tline range"
+								line = Bidi.reorder_line(analysis, line_range) ?? return "FAIL\tline reorder"
+								checksum = checksum + Bidi.visual_to_logical(line).len()
+								line_index = line_index + 1
+							}
+							at = at + 1
+						}
+						"${checksum.to_str()}\t${source.count_utf8_bytes().to_str()}"
+					}
+				}
+				_ => "FAIL\tmalformed benchmark header"
+			}
+		}
+		_ => "FAIL\tmalformed input"
+	}
+}

benchmarks/bidi/run.py

@@ -0,0 +1,123 @@
+#!/usr/bin/env python3
+"""Measure retained UAX #9 paragraph analysis and per-line reordering."""
+
+from __future__ import annotations
+
+import argparse
+import json
+import subprocess
+import time
+from pathlib import Path
+
+
+ROOT = Path(__file__).resolve().parents[2]
+BENCH = Path(__file__).resolve().parent
+BUILD = ROOT / ".roc-unicode-tmp" / "benchmarks" / "bidi"
+
+
+def corpus(target_bytes: int) -> dict[str, str]:
+    def repeated(unit: str) -> str:
+        return unit * max(1, target_bytes // len(unit.encode("utf-8")))
+
+    return {
+        "ltr": repeated("The quick brown fox has 123 words. "),
+        "mixed": repeated("abc אבג 123 العربية "),
+        "neutrals-nsm": repeated("א(̀  ...  )ب "),
+        "nested-isolates": repeated("a \u2068אב \u2066(12)\u2069\u2069 ب "),
+        "overflow-controls": repeated(("\u202b" * 126) + "a" + ("\u202c" * 126)),
+        "repeated-fsi": repeated("a \u2068אב \u2068(12)\u2069\u2069 ب "),
+        "brackets": repeated("א([<abc>])ب "),
+        "brackets-63": repeated(("(" * 63) + "a" + (")" * 63)),
+        "brackets-64": repeated(("(" * 64) + "a" + (")" * 64)),
+        "many-runs": repeated("aא1ب(a)2 "),
+    }
+
+
+def command(args: list[str], *, cwd: Path = ROOT) -> None:
+    print("+", " ".join(args), flush=True)
+    subprocess.run(args, cwd=cwd, check=True)
+
+
+def verify_pinned_roc(roc: str) -> None:
+    """Reject a benchmark run whose compiler is not the repository pin."""
+    pin = (ROOT / ".roc-version").read_text(encoding="utf-8").strip()
+    if not pin:
+        raise RuntimeError(".roc-version is empty")
+    completed = subprocess.run([roc, "version"], text=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, check=True)
+    observed = completed.stdout.strip()
+    revision = pin.rsplit("-", 1)[-1]
+    if pin not in observed and revision not in observed:
+        raise RuntimeError(f"Roc compiler differs from .roc-version ({pin}): {observed}")
+
+
+def build(roc: str, zig: str) -> Path:
+    BUILD.mkdir(parents=True, exist_ok=True)
+    command([zig, "build", "--build-file", "tests/platform/build.zig", "native", "-Doptimize=ReleaseFast"])
+    source = BENCH / "main.roc"
+    binary = BUILD / "bidi"
+    command([roc, "check", str(source), "--no-cache"])
+    command([roc, "build", str(source), "--opt=speed", f"--output={binary}", "--no-cache"])
+    return binary
+
+
+def measure(binary: Path, text: str, repeats: int, line_reorders: int) -> dict[str, float | int]:
+    payload = f"{repeats}\t{line_reorders}\n{text}".encode("utf-8")
+    started = time.perf_counter()
+    completed = subprocess.run([str(binary)], input=payload, stdout=subprocess.PIPE, check=True)
+    elapsed = time.perf_counter() - started
+    checksum_text, byte_text = completed.stdout.decode("utf-8").strip().split("\t")
+    checksum, byte_count = int(checksum_text), int(byte_text)
+    if checksum <= 0 or byte_count != len(text.encode("utf-8")):
+        raise RuntimeError(f"benchmark output drifted: {completed.stdout!r}")
+    return {"bytes": byte_count, "checksum": checksum, "seconds": elapsed}
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--roc", default="roc")
+    parser.add_argument("--zig", default="zig")
+    parser.add_argument("--sizes", default="4096,16384,65536", help="comma-separated corpus byte targets")
+    parser.add_argument("--repeats", type=int, default=3)
+    parser.add_argument("--line-reorders", type=int, default=64, help="logical line partitions reordered per retained analysis")
+    parser.add_argument("--output", type=Path, default=BUILD / "results.json")
+    args = parser.parse_args()
+    try:
+        sizes = tuple(int(value) for value in args.sizes.split(","))
+    except ValueError as error:
+        parser.error(f"--sizes must be comma-separated positive integers: {error}")
+    if not sizes or any(size < 1 for size in sizes) or args.repeats < 1 or args.line_reorders < 1:
+        parser.error("--sizes, --repeats, and --line-reorders must be positive")
+    verify_pinned_roc(args.roc)
+    binary = build(args.roc, args.zig)
+    results = {
+        str(size): {
+            name: measure(binary, text, args.repeats, args.line_reorders)
+            for name, text in corpus(size).items()
+        }
+        for size in sizes
+    }
+    scaling = {}
+    for smaller, larger in zip(sizes, sizes[1:]):
+        scaling[f"{smaller}->{larger}"] = {
+            name: round(results[str(larger)][name]["seconds"] / max(results[str(smaller)][name]["seconds"], 0.000001), 3)
+            for name in corpus(smaller)
+        }
+    args.output.parent.mkdir(parents=True, exist_ok=True)
+    args.output.write_text(
+        json.dumps(
+            {
+                "sizes": sizes,
+                "repeats": args.repeats,
+                "line_reorders": args.line_reorders,
+                "cases": results,
+                "scaling_seconds_ratio": scaling,
+            },
+            indent=2,
+        )
+        + "\n"
+    )
+    print(args.output)
+
+
+if __name__ == "__main__":
+    main()

examples/bidi-analysis.roc

@@ -0,0 +1,106 @@
+app [main!] {
+	pf: platform "https://github.com/roc-lang/basic-cli/releases/download/0.21.0/4rAQg8kUYZ3Vksr4qMQHpaFYNiHSn9GgS7gVxghd1XYV.tar.zst",
+	unicode: "../package/main.roc",
+}
+
+import CliArgs
+import pf.IOErr exposing [IOErr]
+import pf.OsStr exposing [OsStr]
+import pf.Stderr
+import pf.Stdout
+import unicode.Bidi
+import unicode.Scalar
+import unicode.ScalarRange
+import unicode.TextRange
+
+parse_base = |name| match name {
+	"auto" => Ok(Auto)
+	"ltr" => Ok(LeftToRight)
+	"rtl" => Ok(RightToLeft)
+	_ => Err(UnknownBase(name))
+}
+
+base_name = |base| match base {
+	Auto => "auto"
+	LeftToRight => "ltr"
+	RightToLeft => "rtl"
+}
+
+level_name = |level| match level {
+	Level(value) => value.to_str()
+	RemovedByX9 => "x"
+}
+
+scalar_option_name = |value| match value {
+	None => "none"
+	Some(scalar) => Scalar.to_u32(scalar).to_str()
+}
+
+format_mirrors = |mirrors| {
+	var output = []
+	var at = 0.U64
+	for mirror in mirrors {
+		if mirror.needs_glyph {
+			output = output.append("${at.to_str()}:${scalar_option_name(mirror.glyph)}")
+		}
+		at = at + 1
+	}
+	if output.is_empty() "none" else Str.join_with(output, ",")
+}
+
+render = |base, analysis, line| {
+	paragraph = TextRange.scalar_range(Bidi.paragraph_range(analysis))
+	levels = Bidi.line_levels(line).map(level_name)
+	visual = Bidi.visual_to_logical(line).map(U64.to_str)
+	logical_to_visual = Bidi.logical_to_visual(line).map(
+		|position| match position {
+			Some(value) => value.to_str()
+			None => "x"
+		},
+	)
+	\\requested-base: ${base_name(base)}
+	\\paragraph-level: ${Bidi.paragraph_level(analysis).to_str()}
+	\\scalar-range: ${ScalarRange.start(paragraph).to_str()}..${ScalarRange.end(paragraph).to_str()}
+	\\line-levels: ${Str.join_with(levels, ",")}
+	\\visual-to-logical: ${Str.join_with(visual, ",")}
+	\\logical-to-visual: ${Str.join_with(logical_to_visual, ",")}
+	\\mirrored-glyphs: ${format_mirrors(Bidi.line_mirroring(line))}
+}
+
+main! : List(OsStr) => Try({}, [Exit(I32), StderrErr(IOErr), StdoutErr(IOErr), ..])
+main! = |os_args| {
+	args = CliArgs.to_strs!(os_args)?
+	match args {
+		[_app, base_text, source] => {
+			base = match parse_base(base_text) {
+				Ok(value) => value
+				Err(UnknownBase(name)) => {
+					Stderr.line!("error: BASE must be auto, ltr, or rtl; got ${Str.inspect(name)}")?
+					return Err(Exit(2))
+				}
+			}
+			analysis = match Bidi.analyze_paragraph(source, base, Bidi.default_limits) {
+				Ok(value) => value
+				Err(error) => {
+					Stderr.line!("error: bidi analysis failed: ${Str.inspect(error)}")?
+					return Err(Exit(1))
+				}
+			}
+			line_range = TextRange.scalar_range(Bidi.paragraph_range(analysis))
+			line = match Bidi.reorder_line(analysis, line_range) {
+				Ok(value) => value
+				Err(error) => {
+					Stderr.line!("error: bidi line reordering failed: ${Str.inspect(error)}")?
+					return Err(Exit(1))
+				}
+			}
+			Stdout.line!(render(base, analysis, line))?
+			Ok({})
+		}
+		_ => {
+			Stderr.line!("usage: bidi-analysis BASE TEXT")?
+			Stderr.line!("       BASE is auto, ltr, or rtl; TEXT is one P1 paragraph")?
+			Err(Exit(2))
+		}
+	}
+}