wip: in-progress compiler/dbcatalog/bindings changes (checkpoint push)

Author: BlobMaster41

package.json

@@ -80,7 +80,7 @@
     "prepublishOnly": "node scripts/build",
     "watch": "node scripts/build --watch",
     "coverage": "npx c8 -- npm test",
-    "test": "npm run test:parser && npm run test:compiler -- --parallel && npm run test:browser && npm run test:toilconfig && npm run test:transform && npm run test:cli && npm run test:json && npm run test:data && npm run test:dbresolve && npm run test:dbstatic && npm run test:streams",
+    "test": "npm run test:parser && npm run test:compiler -- --parallel && npm run test:browser && npm run test:toilconfig && npm run test:transform && npm run test:cli && npm run test:json && npm run test:data && npm run test:dbresolve && npm run test:dbstatic && npm run test:routekinds && npm run test:streams",
     "test:parser": "node --enable-source-maps tests/parser",
     "test:compiler": "node --enable-source-maps --no-warnings tests/compiler",
     "test:browser": "node --enable-source-maps tests/browser",
@@ -93,6 +93,7 @@
     "test:data": "node tests/data/run.mjs",
     "test:dbresolve": "node tests/dbresolve/run.mjs",
     "test:dbstatic": "node tests/dbstatic/run.mjs",
+    "test:routekinds": "node tests/routekinds/run.mjs",
     "test:streams": "node tests/streams/run.mjs && node tests/streams/catalog.mjs && node tests/streams/codegen.mjs",
     "asbuild": "npm run asbuild:debug && npm run asbuild:release",
     "asbuild:debug": "node bin/toilscript --config src/toilconfig.json --target debug",

src/compiler.ts

@@ -22,6 +22,7 @@ import {
 import {
   buildToilDbCatalog,
   buildToilDbTypes,
+  buildToilDbRouteKinds,
   buildToilSurface,
   buildToilStreamCatalog,
   buildToilDaemonCatalog
@@ -833,6 +834,10 @@ export class Compiler extends DiagnosticEmitter {
     let surface = buildToilSurface(program, targetMode);
     if (surface != null) module.addCustomSection("toil.surface", surface);
     if (targetMode != "cold") {
+      // Request-path DB policy metadata. The edge uses this only to tighten
+      // mutating HTTP methods that the source declared read-only with `@query`.
+      let routeKinds = buildToilDbRouteKinds(program);
+      if (routeKinds != null) module.addCustomSection("toildb.route_kinds", routeKinds);
       // hot or legacy: any @stream class -> toilstream.catalog.
       let streamCat = buildToilStreamCatalog(program);
       if (streamCat != null) module.addCustomSection("toilstream.catalog", streamCat);

src/dbcatalog.ts

@@ -575,6 +575,178 @@ export function buildToilDbTypes(program: Program): Uint8Array | null {
   return w.toBytes();
 }
 
+class RouteKindEntry {
+  method: i32 = 0;
+  kind: i32 = 0;
+  path: string = "";
+}
+
+class RouteInfo {
+  method: i32 = -1;
+  path: string = "";
+}
+
+/** Build the `toildb.route_kinds` section bytes, or `null` when no route needs
+ *  an extra runtime DB-policy clamp. The edge already derives the default kind
+ *  from the trusted HTTP method. This section only carries the stricter source
+ *  signal that the method clamp cannot infer: mutating-method routes explicitly
+ *  declared `@query`.
+ *
+ *  Wire format (LE):
+ *    u16 format_version = 1
+ *    u16 n_routes
+ *    per route:
+ *      u8 method          (same values as runtime Methods / request envelope)
+ *      u8 function_kind   (0 = Query)
+ *      str route_pattern  (same normalized pattern emitted into __toilMatch)
+ */
+export function buildToilDbRouteKinds(program: Program): Uint8Array | null {
+  let routes = new Array<RouteKindEntry>();
+  let sources = program.sources;
+  for (let i = 0, k = sources.length; i < k; ++i) {
+    let source = sources[i];
+    if (source.isLibrary) continue;
+    let statements = source.statements;
+    for (let j = 0, l = statements.length; j < l; ++j) {
+      let statement = statements[j];
+      if (statement.kind != NodeKind.ClassDeclaration) continue;
+      let cls = <ClassDeclaration>statement;
+      let rest = decoOf(cls.decorators, DecoratorKind.Rest);
+      if (rest == null) continue;
+      let prefix = restPrefixOf(rest);
+      let members = cls.members;
+      for (let m = 0, mn = members.length; m < mn; ++m) {
+        let member = members[m];
+        if (member.kind != NodeKind.MethodDeclaration) continue;
+        let method = <MethodDeclaration>member;
+        let routeDeco = routeDecoratorOf(method);
+        if (routeDeco == null) continue;
+        let info = routeInfoOf(routeDeco);
+        if (info == null || isSafeMethod(info.method)) continue;
+        if (explicitRequestKind(method.decorators) != 0) continue;
+        let entry = new RouteKindEntry();
+        entry.method = info.method;
+        entry.kind = 0; // FunctionKind::Query
+        entry.path = joinRoutePath(prefix, info.path);
+        routes.push(entry);
+      }
+    }
+  }
+  if (routes.length == 0) return null;
+  let w = new CatWriter();
+  w.u16(1);
+  w.u16(routes.length);
+  for (let i = 0, k = routes.length; i < k; ++i) {
+    w.u8(routes[i].method);
+    w.u8(routes[i].kind);
+    w.str(routes[i].path);
+  }
+  return w.toBytes();
+}
+
+function explicitRequestKind(decorators: DecoratorNode[] | null): i32 {
+  if (decorators == null) return -1;
+  for (let i = 0, k = decorators.length; i < k; ++i) {
+    let dk = decorators[i].decoratorKind;
+    if (dk == DecoratorKind.Query) return 0;
+    if (dk == DecoratorKind.Action) return 1;
+  }
+  return -1;
+}
+
+function isSafeMethod(method: i32): bool {
+  return method == 0 || method == 5 || method == 6;
+}
+
+function restPrefixOf(deco: DecoratorNode): string {
+  let args = deco.args;
+  if (args == null || args.length == 0) return "";
+  let a = args[0];
+  if (a instanceof StringLiteralExpression) {
+    let s = (<StringLiteralExpression>a).value.trim();
+    if (s.length == 0 || s == "/") return "";
+    if (!s.startsWith("/")) s = "/" + s;
+    return s.replace(/\/+$/, "");
+  }
+  return "";
+}
+
+function routeDecoratorOf(method: MethodDeclaration): DecoratorNode | null {
+  let decos = method.decorators;
+  if (decos == null) return null;
+  for (let i = 0, k = decos.length; i < k; ++i) {
+    switch (decos[i].decoratorKind) {
+      case DecoratorKind.Route:
+      case DecoratorKind.Get:
+      case DecoratorKind.Post:
+      case DecoratorKind.Put:
+      case DecoratorKind.Delete:
+      case DecoratorKind.Patch:
+      case DecoratorKind.Head:
+      case DecoratorKind.Options: return decos[i];
+    }
+  }
+  return null;
+}
+
+function routeInfoOf(deco: DecoratorNode): RouteInfo | null {
+  let out = new RouteInfo();
+  let dk = deco.decoratorKind;
+  if (dk == DecoratorKind.Route) {
+    let args = deco.args;
+    if (args == null || args.length == 0 || !(args[0] instanceof ObjectLiteralExpression)) return null;
+    let obj = <ObjectLiteralExpression>args[0];
+    let mv = objectField(obj, "method");
+    let pv = objectField(obj, "path");
+    if (mv == null || pv == null || !(pv instanceof StringLiteralExpression)) return null;
+    let mName = enumMember(mv);
+    if (mName == null) return null;
+    out.method = methodCode(mName);
+    out.path = (<StringLiteralExpression>pv).value;
+  } else {
+    out.method = verbMethodCode(dk);
+    let args = deco.args;
+    if (args == null || args.length == 0 || !(args[0] instanceof StringLiteralExpression)) return null;
+    out.path = (<StringLiteralExpression>args[0]).value;
+  }
+  return out.method >= 0 ? out : null;
+}
+
+function verbMethodCode(dk: DecoratorKind): i32 {
+  switch (dk) {
+    case DecoratorKind.Post: return 1;
+    case DecoratorKind.Put: return 2;
+    case DecoratorKind.Delete: return 3;
+    case DecoratorKind.Patch: return 4;
+    case DecoratorKind.Head: return 5;
+    case DecoratorKind.Options: return 6;
+    default: return 0;
+  }
+}
+
+function methodCode(name: string): i32 {
+  switch (name) {
+    case "GET": return 0;
+    case "POST": return 1;
+    case "PUT": return 2;
+    case "DELETE": return 3;
+    case "PATCH": return 4;
+    case "HEAD": return 5;
+    case "OPTIONS": return 6;
+    default: return -1;
+  }
+}
+
+function joinRoutePath(prefix: string, routePath: string): string {
+  let r = routePath.trim();
+  if (r.length == 0) r = "/";
+  if (!r.startsWith("/")) r = "/" + r;
+  if (r == "/") return prefix.length > 0 ? prefix : "/";
+  if (r.length > 1) r = r.replace(/\/+$/, "");
+  let full = prefix + r;
+  return full.length == 0 ? "/" : full;
+}
+
 // ===========================================================================
 // Streams + daemon metadata sections (spec 03 sections 6/7/8, Part 5 layouts).
 //

src/program.ts

@@ -2291,6 +2291,9 @@ export class Program extends DiagnosticEmitter {
     let isStatic = declaration.is(CommonFlags.Static);
     let methodTargetMode = this.options.targetMode;
     let acceptedFlags = DecoratorFlags.Inline | DecoratorFlags.Unsafe;
+    // ToilDB function kinds also apply to methods, including @rest route
+    // handlers. The parser's DB checker already walks MethodDeclaration bodies.
+    acceptedFlags |= DecoratorFlags.DbFunction;
     if (!declaration.is(CommonFlags.Generic)) {
       acceptedFlags |= DecoratorFlags.OperatorBinary
                     |  DecoratorFlags.OperatorPrefix

std/assembly/bindings/toildb.ts

@@ -35,8 +35,8 @@ export namespace toildbHost {
   export declare function get(handle: u32, keyPtr: usize, keyLen: i32): i32;
 
   // record bounded multi-get. Input at keysPtr: u32 count + per key (u32 len +
-  // bytes). Result (stashed): u32 count + per item u8 present (+ u32 len + bytes
-  // when present), in request order. Returns the stashed length | negative error.
+  // bytes). Result (stashed): u32 count + per item u8 present
+  // (+ u32 schema_version + u32 len + bytes when present), in request order.
   // @ts-ignore: decorator
   @external("env", "data.get_many")
   export declare function getMany(handle: u32, keysPtr: usize, keysLen: i32): i32;
@@ -148,7 +148,7 @@ export namespace toildbHost {
   ): i32;
 
   // membership.list(limit) -> framed-list length (stashed) | negative error.
-  // The blob is `u32 count` then per member `u32 len + bytes`.
+  // The blob is `u32 count` then per member `u32 schema_version + u32 len + bytes`.
   // @ts-ignore: decorator
   @external("env", "data.membership_list")
   export declare function membershipList(handle: u32, setPtr: usize, setLen: i32, limit: i32): i32;
@@ -216,7 +216,7 @@ export namespace toildbHost {
   ): i32;
 
   // events.latest(limit) -> framed-list length (stashed) | negative error.
-  // The blob is `u32 count` then per event `u32 len + bytes`, newest first.
+  // The blob is `u32 count` then per event `u32 schema_version + u32 len + bytes`, newest first.
   // @ts-ignore: decorator
   @external("env", "data.latest")
   export declare function latest(handle: u32, keyPtr: usize, keyLen: i32, limit: i32): i32;

std/assembly/toildb.ts

@@ -471,7 +471,7 @@ export class Events<K, V> {
   }
 
   /// The newest `limit` events, newest first. Decodes each framed event into a
-  /// `V`. The host frames them as `u32 count` then per event `u32 len + bytes`.
+  /// `V`. The host frames them as `u32 count` then per event `u32 schema_version + u32 len + bytes`.
   latest(key: K, limit: i32): V[] {
     const kb = key.encode();
     const status = toildbHost.latest(this.__handle, kb.dataStart, kb.byteLength, limit);