From 802ec47300f19662ff694eac86232ce68726c6ec Mon Sep 17 00:00:00 2001 From: Ben Asher Date: Sun, 28 Jun 2026 14:04:13 -0700 Subject: [PATCH 01/20] Add native N-API backend for libpg_query (PG 18) Native Node addon wrapping libpg_query via N-API, as a memory-efficient alternative to the WASM build. Ships prebuilt platform binaries via esbuild-style optional dependencies (@ashbyhq/libpg-query-native-), so consumers never need node-gyp. API is a drop-in replacement for @libpg-query/parser: parse, fingerprint, normalize, scan, parsePlPgSQL (sync + async variants). For optimal memory behavior with large queries, jemalloc should be preloaded via LD_PRELOAD (Linux) or DYLD_INSERT_LIBRARIES (macOS). Includes memory and throughput benchmarks comparing allocators. Co-Authored-By: Claude Opus 4.6 --- native/.gitignore | 5 + native/Makefile | 98 +++++++++++ native/README.md | 121 +++++++++++++ native/benchmark/memory.mjs | 206 ++++++++++++++++++++++ native/benchmark/run-with-jemalloc.sh | 57 +++++++ native/package.json | 57 +++++++ native/scripts/package-platforms.mjs | 86 ++++++++++ native/src/addon.cc | 235 ++++++++++++++++++++++++++ native/src/index.ts | 177 +++++++++++++++++++ native/test/errors.test.js | 122 +++++++++++++ native/test/fingerprint.test.js | 57 +++++++ native/test/normalize.test.js | 61 +++++++ native/test/parsing.test.js | 76 +++++++++ native/test/plpgsql.test.js | 65 +++++++ native/test/scan.test.js | 88 ++++++++++ native/tsconfig.json | 15 ++ pnpm-workspace.yaml | 1 + 17 files changed, 1527 insertions(+) create mode 100644 native/.gitignore create mode 100644 native/Makefile create mode 100644 native/README.md create mode 100644 native/benchmark/memory.mjs create mode 100755 native/benchmark/run-with-jemalloc.sh create mode 100644 native/package.json create mode 100644 native/scripts/package-platforms.mjs create mode 100644 native/src/addon.cc create mode 100644 native/src/index.ts create mode 100644 native/test/errors.test.js create mode 100644 native/test/fingerprint.test.js create mode 100644 native/test/normalize.test.js create mode 100644 native/test/parsing.test.js create mode 100644 native/test/plpgsql.test.js create mode 100644 native/test/scan.test.js create mode 100644 native/tsconfig.json diff --git a/native/.gitignore b/native/.gitignore new file mode 100644 index 0000000..67b4883 --- /dev/null +++ b/native/.gitignore @@ -0,0 +1,5 @@ +.cache/ +prebuilds/ +packages/ +dist/ +node_modules/ diff --git a/native/Makefile b/native/Makefile new file mode 100644 index 0000000..15f98e8 --- /dev/null +++ b/native/Makefile @@ -0,0 +1,98 @@ +LIBPG_QUERY_REPO := https://github.com/pganalyze/libpg_query.git +LIBPG_QUERY_TAG := 18.0.0 + +OS := $(shell uname -s) +ARCH := $(shell uname -m) + +ifeq ($(OS),Darwin) +PLATFORM := darwin +else ifeq ($(OS),Linux) +PLATFORM := linux +else +$(error Unsupported platform: $(OS)) +endif + +ifeq ($(ARCH),x86_64) +NODE_ARCH := x64 +else ifeq ($(ARCH),aarch64) +NODE_ARCH := arm64 +else ifeq ($(ARCH),arm64) +NODE_ARCH := arm64 +else +NODE_ARCH := $(ARCH) +endif + +MUSL := $(shell ldd --version 2>&1 | grep -c musl 2>/dev/null || echo 0) +ifeq ($(MUSL),1) +PLATFORM_SUFFIX := -musl +else +PLATFORM_SUFFIX := +endif + +PLATFORM_KEY := $(PLATFORM)-$(NODE_ARCH)$(PLATFORM_SUFFIX) +CACHE_DIR := .cache/$(PLATFORM_KEY) +LIBPG_QUERY_DIR := $(CACHE_DIR)/libpg_query + +LIBPG_QUERY_LIB := $(LIBPG_QUERY_DIR)/libpg_query.a + +NODE_ADDON_API_DIR := $(shell node -p "require('node-addon-api').include_dir" 2>/dev/null || echo "node_modules/node-addon-api") +NODE_INCLUDE := $(shell node -p "require('node-api-headers').include_dir" 2>/dev/null || echo "") + +OUT_DIR := prebuilds/$(PLATFORM_KEY) +OUT_FILE := $(OUT_DIR)/libpg_query_native.node + +CXXFLAGS := -O2 -std=c++17 -fPIC -DNAPI_VERSION=8 -DNODE_ADDON_API_DISABLE_DEPRECATED +LDFLAGS := + +ifeq ($(PLATFORM),darwin) +LDFLAGS += -undefined dynamic_lookup -dynamiclib +else +LDFLAGS += -shared +endif + +INCLUDES := \ + -I$(LIBPG_QUERY_DIR) \ + -I$(LIBPG_QUERY_DIR)/vendor \ + -I$(NODE_ADDON_API_DIR) \ + $(if $(NODE_INCLUDE),-I$(NODE_INCLUDE),) + +# Clone and build libpg_query +$(LIBPG_QUERY_DIR): + mkdir -p $(CACHE_DIR) + git clone -b $(LIBPG_QUERY_TAG) --single-branch --depth 1 $(LIBPG_QUERY_REPO) $(LIBPG_QUERY_DIR) + +$(LIBPG_QUERY_LIB): $(LIBPG_QUERY_DIR) + cd $(LIBPG_QUERY_DIR) && $(MAKE) build + +# Build the .node addon +$(OUT_FILE): src/addon.cc $(LIBPG_QUERY_LIB) + mkdir -p $(OUT_DIR) + $(CXX) $(CXXFLAGS) $(INCLUDES) \ + src/addon.cc \ + $(LIBPG_QUERY_LIB) \ + $(LDFLAGS) \ + -lpthread \ + -o $(OUT_FILE) + @echo "Built $(OUT_FILE) ($$(du -h $(OUT_FILE) | cut -f1))" + +.PHONY: build clean clean-cache deps info + +build: $(OUT_FILE) + +deps: $(LIBPG_QUERY_LIB) + +clean: + rm -rf prebuilds + +clean-cache: + rm -rf .cache + +info: + @echo "Platform: $(PLATFORM_KEY)" + @echo "libpg_query: $(LIBPG_QUERY_TAG)" + @echo "Output: $(OUT_FILE)" + @echo "Node API dir: $(NODE_ADDON_API_DIR)" + @echo "" + @echo "For jemalloc memory benefits, use LD_PRELOAD:" + @echo " Linux: LD_PRELOAD=/usr/lib/x86_64-linux-gnu/libjemalloc.so node app.js" + @echo " macOS: DYLD_INSERT_LIBRARIES=/opt/homebrew/lib/libjemalloc.dylib node app.js" diff --git a/native/README.md b/native/README.md new file mode 100644 index 0000000..f001712 --- /dev/null +++ b/native/README.md @@ -0,0 +1,121 @@ +# @ashbyhq/libpg-query-native + +Native N-API PostgreSQL query parser — a memory-efficient alternative to the WASM build. + +## Why native? + +The WASM build (`@libpg-query/parser`) has unavoidable memory behavior for large queries: +- Peak RSS ~935 MB for a 2.74 MB query (vs ~70 MB native with jemalloc) +- WebAssembly linear memory never shrinks — RSS ratchets up permanently +- No allocator can fix this within WASM constraints + +The native build eliminates both problems. With jemalloc preloaded, RSS tracks the live set and returns to baseline after each parse. + +``` + idle big-query peak ratchet throughput +WASM (today) 32 MB 935 MB permanent 126k/s +Native ~4 MB ~70 MB* none* 259k/s +``` + +\* With jemalloc. Without jemalloc, macOS libmalloc retains ~274 MB (still 3.5x better than WASM). + +## Installation + +```bash +npm install @ashbyhq/libpg-query-native +``` + +Platform-specific binaries are installed automatically via optional dependencies: +- `@ashbyhq/libpg-query-native-darwin-arm64` +- `@ashbyhq/libpg-query-native-darwin-x64` +- `@ashbyhq/libpg-query-native-linux-x64` +- `@ashbyhq/libpg-query-native-linux-arm64` +- `@ashbyhq/libpg-query-native-linux-x64-musl` +- `@ashbyhq/libpg-query-native-linux-arm64-musl` + +No node-gyp or compiler toolchain needed at install time. + +## Usage + +Drop-in replacement for `@libpg-query/parser`: + +```js +const { parse, parseSync, fingerprint, normalize, scan } = require('@ashbyhq/libpg-query-native'); + +// Sync (no init needed — native loads instantly) +const result = parseSync('SELECT id, name FROM users WHERE active = true'); + +// Async (same result, just wrapped in a promise) +const result2 = await parse('SELECT id, name FROM users WHERE active = true'); +``` + +### API + +| Function | Sync | Async | Returns | +|----------|------|-------|---------| +| `parseSync(sql)` / `parse(sql)` | ✓ | ✓ | `ParseResult` (JSON AST) | +| `parsePlPgSQLSync(sql)` / `parsePlPgSQL(sql)` | ✓ | ✓ | PL/pgSQL parse tree | +| `fingerprintSync(sql)` / `fingerprint(sql)` | ✓ | ✓ | 16-char hex fingerprint | +| `normalizeSync(sql)` / `normalize(sql)` | ✓ | ✓ | Normalized query string | +| `scanSync(sql)` / `scan(sql)` | ✓ | ✓ | `ScanResult` with tokens | + +## Using jemalloc for optimal memory + +The native addon uses the system allocator by default. For optimal memory behavior +(especially with large queries), preload jemalloc: + +```bash +# Linux +LD_PRELOAD=/usr/lib/x86_64-linux-gnu/libjemalloc.so.2 node app.js + +# macOS (brew install jemalloc) +DYLD_INSERT_LIBRARIES=$(brew --prefix jemalloc)/lib/libjemalloc.dylib node app.js +``` + +Or in your Dockerfile: +```dockerfile +RUN apt-get install -y libjemalloc2 +ENV LD_PRELOAD=/usr/lib/x86_64-linux-gnu/libjemalloc.so.2 +``` + +## Benchmarks + +```bash +# Quick sanity check +node --expose-gc benchmark/memory.mjs --small + +# Full benchmark (large query, ~2.74 MB SQL) +node --expose-gc benchmark/memory.mjs + +# Compare with/without jemalloc +bash benchmark/run-with-jemalloc.sh + +# Throughput benchmark +node --expose-gc benchmark/memory.mjs --throughput + +# Compare against WASM (requires @libpg-query/parser installed) +node --expose-gc benchmark/memory.mjs --all +``` + +## Building from source + +```bash +cd native +npm install +make build # builds libpg_query + the .node addon +npm run build:ts # compiles TypeScript +npm test +``` + +### Generating platform packages + +After building: +```bash +node scripts/package-platforms.mjs +``` + +This creates `packages/libpg-query-native-/` directories ready for `npm publish`. + +## License + +MIT diff --git a/native/benchmark/memory.mjs b/native/benchmark/memory.mjs new file mode 100644 index 0000000..2b75a37 --- /dev/null +++ b/native/benchmark/memory.mjs @@ -0,0 +1,206 @@ +/** + * Memory benchmark: compare allocator behavior for large SQL parses. + * + * Generates a ~2.74 MB SQL query (1500× UNION ALL of SELECT c0..c399 FROM t), + * parses it, and measures RSS at idle, peak (post-parse), and after free. + * + * Usage: + * node benchmark/memory.mjs # native (jemalloc) + * node benchmark/memory.mjs --wasm # WASM backend for comparison + * node benchmark/memory.mjs --system-malloc # native without jemalloc + * node benchmark/memory.mjs --all # run all available backends + * node benchmark/memory.mjs --small # quick sanity check (small query) + * node benchmark/memory.mjs --throughput # throughput benchmark + */ + +import { argv } from "process"; + +const MB = 1024 * 1024; + +function rss() { + return process.memoryUsage.rss(); +} + +function rssMB() { + return (rss() / MB).toFixed(1); +} + +function generateBigQuery(unions = 1500, cols = 400) { + const colList = Array.from({ length: cols }, (_, i) => `c${i}`).join(", "); + const select = `SELECT ${colList} FROM t`; + return Array.from({ length: unions }, () => select).join(" UNION ALL "); +} + +function generateSmallQuery() { + return "SELECT id, name, email, created_at FROM users WHERE id = $1 AND active = true"; +} + +async function benchmarkParse(backend, label, query) { + // Force GC if available + if (global.gc) global.gc(); + await new Promise((r) => setTimeout(r, 100)); + + const idleRss = rss(); + console.log(`\n--- ${label} ---`); + console.log(`Query size: ${(query.length / MB).toFixed(2)} MB`); + console.log(`Idle RSS: ${(idleRss / MB).toFixed(1)} MB`); + + const start = performance.now(); + let result; + try { + result = await backend.parse(query); + } catch (e) { + console.log(`Parse failed: ${e.message}`); + return null; + } + const parseTime = performance.now() - start; + + const peakRss = rss(); + const resultJson = JSON.stringify(result); + const resultSize = Buffer.byteLength(resultJson); + + console.log(`Parse time: ${parseTime.toFixed(0)} ms`); + console.log(`Result size: ${(resultSize / MB).toFixed(1)} MB`); + console.log(`Peak RSS: ${(peakRss / MB).toFixed(1)} MB`); + console.log(`Peak delta: +${((peakRss - idleRss) / MB).toFixed(1)} MB`); + + // Drop references and let GC + allocator return memory + result = null; + if (global.gc) global.gc(); + await new Promise((r) => setTimeout(r, 500)); + + const afterRss = rss(); + console.log(`After free: ${(afterRss / MB).toFixed(1)} MB`); + console.log(`Retained: +${((afterRss - idleRss) / MB).toFixed(1)} MB`); + + return { + label, + idleRss, + peakRss, + afterRss, + parseTime, + resultSize, + querySize: query.length, + }; +} + +async function benchmarkThroughput(backend, label, iterations = 10000) { + const query = "SELECT id, name FROM users WHERE id = $1"; + + // Warmup + for (let i = 0; i < 100; i++) { + backend.parseSync(query); + } + + const start = performance.now(); + for (let i = 0; i < iterations; i++) { + backend.parseSync(query); + } + const elapsed = performance.now() - start; + const opsPerSec = Math.round((iterations / elapsed) * 1000); + + console.log(`\n--- ${label} throughput ---`); + console.log(`${iterations} parses in ${elapsed.toFixed(0)} ms`); + console.log(`${opsPerSec.toLocaleString()} ops/sec`); + + return { label, iterations, elapsed, opsPerSec }; +} + +async function loadNative() { + try { + return await import("../dist/index.js"); + } catch { + console.error( + "Native backend not found. Run 'make build && npm run build:ts' first." + ); + return null; + } +} + +async function loadWasm() { + try { + const wasm = await import("@libpg-query/parser"); + if (wasm.loadModule) await wasm.loadModule(); + return wasm; + } catch { + console.error( + "WASM backend not found. Install @libpg-query/parser for comparison." + ); + return null; + } +} + +function isJemallocLoaded() { + return !!(process.env.LD_PRELOAD?.includes("jemalloc") || + process.env.DYLD_INSERT_LIBRARIES?.includes("jemalloc")); +} + +async function main() { + const flags = new Set(argv.slice(2)); + + const runAll = flags.has("--all"); + const runWasm = flags.has("--wasm") || runAll; + const runNative = !flags.has("--wasm") || runAll; + const small = flags.has("--small"); + const throughput = flags.has("--throughput"); + + const query = small ? generateSmallQuery() : generateBigQuery(); + + console.log("=== libpg-query Memory Benchmark ==="); + console.log(`Node ${process.version}, ${process.platform}-${process.arch}`); + console.log(`GC exposed: ${typeof global.gc === "function"}`); + if (typeof global.gc !== "function") { + console.log("Hint: run with --expose-gc for more accurate measurements"); + } + + const results = []; + + if (runNative) { + const native = await loadNative(); + if (native) { + const jemallocLoaded = isJemallocLoaded(); + const label = jemallocLoaded ? "Native (jemalloc)" : "Native (system malloc)"; + results.push(await benchmarkParse(native, label, query)); + if (throughput) { + await benchmarkThroughput(native, label); + } + } + } + + if (runWasm) { + const wasm = await loadWasm(); + if (wasm) { + results.push(await benchmarkParse(wasm, "WASM (emscripten)", query)); + if (throughput) { + await benchmarkThroughput(wasm, "WASM (emscripten)"); + } + } + } + + // Summary table + const valid = results.filter(Boolean); + if (valid.length > 1) { + console.log("\n=== Summary ==="); + console.log( + "Backend".padEnd(25) + + "Idle".padStart(10) + + "Peak".padStart(10) + + "After".padStart(10) + + "Retained".padStart(10) + + "Parse ms".padStart(10) + ); + console.log("-".repeat(75)); + for (const r of valid) { + console.log( + r.label.padEnd(25) + + `${(r.idleRss / MB).toFixed(1)} MB`.padStart(10) + + `${(r.peakRss / MB).toFixed(1)} MB`.padStart(10) + + `${(r.afterRss / MB).toFixed(1)} MB`.padStart(10) + + `+${((r.afterRss - r.idleRss) / MB).toFixed(1)} MB`.padStart(10) + + `${r.parseTime.toFixed(0)}`.padStart(10) + ); + } + } +} + +main().catch(console.error); diff --git a/native/benchmark/run-with-jemalloc.sh b/native/benchmark/run-with-jemalloc.sh new file mode 100755 index 0000000..d6174b4 --- /dev/null +++ b/native/benchmark/run-with-jemalloc.sh @@ -0,0 +1,57 @@ +#!/bin/bash +# Run the memory benchmark with jemalloc preloaded. +# +# Prerequisites: +# macOS: brew install jemalloc +# Linux: apt install libjemalloc-dev OR yum install jemalloc-devel + +set -e + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" + +# Find jemalloc +if [[ "$(uname)" == "Darwin" ]]; then + JEMALLOC_LIB="${JEMALLOC_LIB:-$(brew --prefix jemalloc 2>/dev/null)/lib/libjemalloc.dylib}" + if [[ ! -f "$JEMALLOC_LIB" ]]; then + echo "jemalloc not found. Install with: brew install jemalloc" + echo "Or set JEMALLOC_LIB=/path/to/libjemalloc.dylib" + exit 1 + fi + echo "Using jemalloc: $JEMALLOC_LIB" + echo "" + + echo "=== Without jemalloc ===" + node --expose-gc "$SCRIPT_DIR/memory.mjs" "$@" + echo "" + echo "=== With jemalloc (DYLD_INSERT_LIBRARIES) ===" + DYLD_INSERT_LIBRARIES="$JEMALLOC_LIB" node --expose-gc "$SCRIPT_DIR/memory.mjs" "$@" +else + JEMALLOC_LIB="${JEMALLOC_LIB:-}" + if [[ -z "$JEMALLOC_LIB" ]]; then + for candidate in \ + /usr/lib/x86_64-linux-gnu/libjemalloc.so.2 \ + /usr/lib/aarch64-linux-gnu/libjemalloc.so.2 \ + /usr/lib64/libjemalloc.so.2 \ + /usr/lib/libjemalloc.so.2 \ + /usr/lib/x86_64-linux-gnu/libjemalloc.so \ + /usr/lib/libjemalloc.so; do + if [[ -f "$candidate" ]]; then + JEMALLOC_LIB="$candidate" + break + fi + done + fi + if [[ -z "$JEMALLOC_LIB" || ! -f "$JEMALLOC_LIB" ]]; then + echo "jemalloc not found. Install with: apt install libjemalloc-dev" + echo "Or set JEMALLOC_LIB=/path/to/libjemalloc.so" + exit 1 + fi + echo "Using jemalloc: $JEMALLOC_LIB" + echo "" + + echo "=== Without jemalloc ===" + node --expose-gc "$SCRIPT_DIR/memory.mjs" "$@" + echo "" + echo "=== With jemalloc (LD_PRELOAD) ===" + LD_PRELOAD="$JEMALLOC_LIB" node --expose-gc "$SCRIPT_DIR/memory.mjs" "$@" +fi diff --git a/native/package.json b/native/package.json new file mode 100644 index 0000000..d8247b1 --- /dev/null +++ b/native/package.json @@ -0,0 +1,57 @@ +{ + "name": "@ashbyhq/libpg-query-native", + "version": "0.1.0", + "description": "PostgreSQL query parser — native N-API + jemalloc backend", + "main": "./dist/index.js", + "types": "./dist/index.d.ts", + "exports": { + ".": { + "import": "./dist/index.js", + "require": "./dist/index.js", + "types": "./dist/index.d.ts" + } + }, + "files": [ + "dist/**/*" + ], + "scripts": { + "build:native": "make build", + "build:ts": "tsc", + "build": "npm run build:native && npm run build:ts", + "clean": "make clean && rm -rf dist", + "test": "node --test test/parsing.test.js test/fingerprint.test.js test/normalize.test.js test/plpgsql.test.js test/scan.test.js test/errors.test.js", + "benchmark": "node benchmark/memory.mjs", + "package-platforms": "node scripts/package-platforms.mjs" + }, + "keywords": [ + "postgresql", + "parser", + "sql", + "native", + "napi", + "jemalloc" + ], + "license": "MIT", + "repository": { + "type": "git", + "url": "git://github.com/ashbyhq/libpg-query-node.git", + "directory": "native" + }, + "dependencies": { + "@pgsql/types": "^18.0.0" + }, + "devDependencies": { + "@types/node": "^26.0.1", + "node-addon-api": "^8.3.1", + "node-api-headers": "^1.5.0", + "typescript": "^5.3.3" + }, + "optionalDependencies": { + "@ashbyhq/libpg-query-native-darwin-arm64": "0.1.0", + "@ashbyhq/libpg-query-native-darwin-x64": "0.1.0", + "@ashbyhq/libpg-query-native-linux-x64": "0.1.0", + "@ashbyhq/libpg-query-native-linux-arm64": "0.1.0", + "@ashbyhq/libpg-query-native-linux-x64-musl": "0.1.0", + "@ashbyhq/libpg-query-native-linux-arm64-musl": "0.1.0" + } +} diff --git a/native/scripts/package-platforms.mjs b/native/scripts/package-platforms.mjs new file mode 100644 index 0000000..2ef8af2 --- /dev/null +++ b/native/scripts/package-platforms.mjs @@ -0,0 +1,86 @@ +/** + * Generate per-platform npm packages from prebuilds. + * Each package contains just the .node binary and a package.json. + * + * Run after `make build` to create packages/ directory: + * node scripts/package-platforms.mjs + */ + +import { readdirSync, mkdirSync, copyFileSync, writeFileSync, existsSync } from "fs"; +import { join, basename } from "path"; + +const VERSION = "0.1.0"; +const SCOPE = "@ashbyhq"; +const BASE_NAME = "libpg-query-native"; +const PREBUILDS_DIR = "prebuilds"; +const OUT_DIR = "packages"; + +const PLATFORM_META = { + "darwin-arm64": { os: ["darwin"], cpu: ["arm64"] }, + "darwin-x64": { os: ["darwin"], cpu: ["x64"] }, + "linux-x64": { os: ["linux"], cpu: ["x64"] }, + "linux-arm64": { os: ["linux"], cpu: ["arm64"] }, + "linux-x64-musl": { os: ["linux"], cpu: ["x64"], libc: ["musl"] }, + "linux-arm64-musl": { os: ["linux"], cpu: ["arm64"], libc: ["musl"] }, +}; + +if (!existsSync(PREBUILDS_DIR)) { + console.error(`No ${PREBUILDS_DIR}/ directory found. Run 'make build' first.`); + process.exit(1); +} + +const platforms = readdirSync(PREBUILDS_DIR); + +for (const platform of platforms) { + const meta = PLATFORM_META[platform]; + if (!meta) { + console.warn(`Skipping unknown platform: ${platform}`); + continue; + } + + const pkgName = `${SCOPE}/${BASE_NAME}-${platform}`; + const pkgDir = join(OUT_DIR, `${BASE_NAME}-${platform}`); + + mkdirSync(pkgDir, { recursive: true }); + + // Copy the .node file + const nodeFile = join(PREBUILDS_DIR, platform, "libpg_query_native.node"); + if (!existsSync(nodeFile)) { + console.warn(`No .node file for ${platform}, skipping`); + continue; + } + copyFileSync(nodeFile, join(pkgDir, "libpg_query_native.node")); + + // Write index.js that just re-exports the binary + writeFileSync( + join(pkgDir, "index.js"), + `module.exports = require('./libpg_query_native.node');\n` + ); + + // Write package.json + const pkg = { + name: pkgName, + version: VERSION, + description: `libpg-query native addon for ${platform}`, + main: "index.js", + files: ["index.js", "libpg_query_native.node"], + os: meta.os, + cpu: meta.cpu, + ...(meta.libc ? { libc: meta.libc } : {}), + license: "MIT", + repository: { + type: "git", + url: "git://github.com/ashbyhq/libpg-query-node.git", + directory: `native/packages/${BASE_NAME}-${platform}`, + }, + publishConfig: { + access: "public", + }, + }; + + writeFileSync(join(pkgDir, "package.json"), JSON.stringify(pkg, null, 2) + "\n"); + + console.log(`Created ${pkgName} (${nodeFile})`); +} + +console.log(`\nDone. Platform packages in ${OUT_DIR}/`); diff --git a/native/src/addon.cc b/native/src/addon.cc new file mode 100644 index 0000000..5574619 --- /dev/null +++ b/native/src/addon.cc @@ -0,0 +1,235 @@ +#include +#include + +extern "C" { +#include "pg_query.h" +#include "protobuf/pg_query.pb-c.h" +} + +static std::string EscapeJsonString(const std::string &s) { + std::string out; + out.reserve(s.size() + 8); + for (char c : s) { + switch (c) { + case '"': out += "\\\""; break; + case '\\': out += "\\\\"; break; + case '\n': out += "\\n"; break; + case '\r': out += "\\r"; break; + case '\t': out += "\\t"; break; + default: out += c; + } + } + return out; +} + +// Build a JSON object with error details from a PgQueryError. +static std::string BuildErrorJson(PgQueryError *error) { + std::string msg = error->message ? error->message : "Unknown error"; + std::string json = "{\"message\":\"" + EscapeJsonString(msg) + "\""; + json += ",\"cursorPosition\":" + std::to_string(error->cursorpos > 0 ? error->cursorpos - 1 : 0); + if (error->funcname) + json += ",\"functionName\":\"" + EscapeJsonString(error->funcname) + "\""; + if (error->filename) + json += ",\"fileName\":\"" + EscapeJsonString(error->filename) + "\""; + if (error->lineno > 0) + json += ",\"lineNumber\":" + std::to_string(error->lineno); + if (error->context) + json += ",\"context\":\"" + EscapeJsonString(error->context) + "\""; + json += "}"; + return json; +} + +// Return a {error: json, result: null} object to JS. JS throws from there. +static Napi::Value ReturnError(Napi::Env env, PgQueryError *error) { + Napi::Object obj = Napi::Object::New(env); + obj.Set("error", Napi::String::New(env, BuildErrorJson(error))); + obj.Set("result", env.Null()); + return obj; +} + +static Napi::Value ReturnResult(Napi::Env env, const std::string &result) { + Napi::Object obj = Napi::Object::New(env); + obj.Set("error", env.Null()); + obj.Set("result", Napi::String::New(env, result)); + return obj; +} + +static std::string ValidateQuery(Napi::Env env, const Napi::CallbackInfo &info) { + if (info.Length() < 1 || !info[0].IsString()) { + Napi::TypeError::New(env, "Expected a string argument").ThrowAsJavaScriptException(); + return ""; + } + std::string query = info[0].As().Utf8Value(); + if (query.empty()) { + Napi::Error::New(env, "Query cannot be empty").ThrowAsJavaScriptException(); + return ""; + } + return query; +} + +static Napi::Value ParseSync(const Napi::CallbackInfo &info) { + Napi::Env env = info.Env(); + std::string query = ValidateQuery(env, info); + if (env.IsExceptionPending()) return env.Undefined(); + + PgQueryParseResult result = pg_query_parse(query.c_str()); + + if (result.error) { + Napi::Value ret = ReturnError(env, result.error); + pg_query_free_parse_result(result); + return ret; + } + + std::string json(result.parse_tree); + pg_query_free_parse_result(result); + return ReturnResult(env, json); +} + +static Napi::Value ParsePlPgSQLSync(const Napi::CallbackInfo &info) { + Napi::Env env = info.Env(); + std::string query = ValidateQuery(env, info); + if (env.IsExceptionPending()) return env.Undefined(); + + PgQueryPlpgsqlParseResult result = pg_query_parse_plpgsql(query.c_str()); + + if (result.error) { + Napi::Value ret = ReturnError(env, result.error); + pg_query_free_plpgsql_parse_result(result); + return ret; + } + + std::string json = std::string("{\"plpgsql_funcs\":") + + (result.plpgsql_funcs ? result.plpgsql_funcs : "[]") + "}"; + pg_query_free_plpgsql_parse_result(result); + return ReturnResult(env, json); +} + +static Napi::Value FingerprintSync(const Napi::CallbackInfo &info) { + Napi::Env env = info.Env(); + std::string query = ValidateQuery(env, info); + if (env.IsExceptionPending()) return env.Undefined(); + + PgQueryFingerprintResult result = pg_query_fingerprint(query.c_str()); + + if (result.error) { + Napi::Value ret = ReturnError(env, result.error); + pg_query_free_fingerprint_result(result); + return ret; + } + + std::string fp(result.fingerprint_str); + pg_query_free_fingerprint_result(result); + return ReturnResult(env, fp); +} + +static Napi::Value NormalizeSync(const Napi::CallbackInfo &info) { + Napi::Env env = info.Env(); + std::string query = ValidateQuery(env, info); + if (env.IsExceptionPending()) return env.Undefined(); + + PgQueryNormalizeResult result = pg_query_normalize(query.c_str()); + + if (result.error) { + Napi::Value ret = ReturnError(env, result.error); + pg_query_free_normalize_result(result); + return ret; + } + + std::string normalized(result.normalized_query); + pg_query_free_normalize_result(result); + return ReturnResult(env, normalized); +} + +static std::string GetTokenName(int token_type) { + switch (token_type) { + case 258: return "IDENT"; + case 261: return "SCONST"; + case 266: return "ICONST"; + case 260: return "FCONST"; + case 267: return "PARAM"; + case 40: return "ASCII_40"; + case 41: return "ASCII_41"; + case 42: return "ASCII_42"; + case 44: return "ASCII_44"; + case 59: return "ASCII_59"; + case 61: return "ASCII_61"; + case 268: return "TYPECAST"; + case 272: return "LESS_EQUALS"; + case 273: return "GREATER_EQUALS"; + case 274: return "NOT_EQUALS"; + case 275: return "SQL_COMMENT"; + case 276: return "C_COMMENT"; + default: return "UNKNOWN"; + } +} + +static std::string GetKeywordName(int kind) { + switch (kind) { + case 0: return "NO_KEYWORD"; + case 1: return "UNRESERVED_KEYWORD"; + case 2: return "COL_NAME_KEYWORD"; + case 3: return "TYPE_FUNC_NAME_KEYWORD"; + case 4: return "RESERVED_KEYWORD"; + default: return "UNKNOWN_KEYWORD"; + } +} + +static Napi::Value ScanSync(const Napi::CallbackInfo &info) { + Napi::Env env = info.Env(); + std::string query = ValidateQuery(env, info); + if (env.IsExceptionPending()) return env.Undefined(); + + PgQueryScanResult result = pg_query_scan(query.c_str()); + + if (result.error) { + Napi::Value ret = ReturnError(env, result.error); + pg_query_free_scan_result(result); + return ret; + } + + PgQuery__ScanResult *scan_result = pg_query__scan_result__unpack( + NULL, result.pbuf.len, (const uint8_t *)result.pbuf.data); + + if (!scan_result) { + pg_query_free_scan_result(result); + Napi::Error::New(env, "Failed to unpack scan result").ThrowAsJavaScriptException(); + return env.Undefined(); + } + + Napi::Object scan_obj = Napi::Object::New(env); + scan_obj.Set("version", Napi::Number::New(env, scan_result->version)); + + Napi::Array tokens = Napi::Array::New(env, scan_result->n_tokens); + for (size_t i = 0; i < scan_result->n_tokens; i++) { + PgQuery__ScanToken *token = scan_result->tokens[i]; + Napi::Object tok = Napi::Object::New(env); + tok.Set("start", Napi::Number::New(env, token->start)); + tok.Set("end", Napi::Number::New(env, token->end)); + tok.Set("text", Napi::String::New(env, query.substr(token->start, token->end - token->start))); + tok.Set("tokenType", Napi::Number::New(env, token->token)); + tok.Set("tokenName", Napi::String::New(env, GetTokenName(token->token))); + tok.Set("keywordKind", Napi::Number::New(env, token->keyword_kind)); + tok.Set("keywordName", Napi::String::New(env, GetKeywordName(token->keyword_kind))); + tokens[i] = tok; + } + scan_obj.Set("tokens", tokens); + + pg_query__scan_result__free_unpacked(scan_result, NULL); + pg_query_free_scan_result(result); + + Napi::Object obj = Napi::Object::New(env); + obj.Set("error", env.Null()); + obj.Set("result", scan_obj); + return obj; +} + +Napi::Object Init(Napi::Env env, Napi::Object exports) { + exports.Set("parseSync", Napi::Function::New(env, ParseSync)); + exports.Set("parsePlPgSQLSync", Napi::Function::New(env, ParsePlPgSQLSync)); + exports.Set("fingerprintSync", Napi::Function::New(env, FingerprintSync)); + exports.Set("normalizeSync", Napi::Function::New(env, NormalizeSync)); + exports.Set("scanSync", Napi::Function::New(env, ScanSync)); + return exports; +} + +NODE_API_MODULE(libpg_query_native, Init) diff --git a/native/src/index.ts b/native/src/index.ts new file mode 100644 index 0000000..bc9901b --- /dev/null +++ b/native/src/index.ts @@ -0,0 +1,177 @@ +import type { ParseResult } from "@pgsql/types"; + +export type { ParseResult } from "@pgsql/types"; + +export interface ScanToken { + start: number; + end: number; + text: string; + tokenType: number; + tokenName: string; + keywordKind: number; + keywordName: string; +} + +export interface ScanResult { + version: number; + tokens: ScanToken[]; +} + +export interface SqlErrorDetails { + message: string; + cursorPosition: number; + fileName?: string; + functionName?: string; + lineNumber?: number; + context?: string; +} + +export class SqlError extends Error { + sqlDetails?: SqlErrorDetails; + + constructor(message: string, details?: SqlErrorDetails) { + super(message); + this.name = "SqlError"; + this.sqlDetails = details; + } +} + +export function hasSqlDetails(error: unknown): error is SqlError { + return error instanceof SqlError && error.sqlDetails !== undefined; +} + +function loadNativeAddon(): NativeAddon { + const platform = process.platform; + const arch = process.arch; + const musl = isMusl(); + + const platformKey = musl ? `${platform}-${arch}-musl` : `${platform}-${arch}`; + + // Try loading from a local prebuild first (development / monorepo) + const path = require("path"); + const localPrebuild = path.join( + __dirname, + "..", + "prebuilds", + platformKey, + "libpg_query_native.node" + ); + try { + return require(localPrebuild); + } catch { + // fall through to platform package + } + + const packageMap: Record = { + "darwin-arm64": "@ashbyhq/libpg-query-native-darwin-arm64", + "darwin-x64": "@ashbyhq/libpg-query-native-darwin-x64", + "linux-x64": "@ashbyhq/libpg-query-native-linux-x64", + "linux-arm64": "@ashbyhq/libpg-query-native-linux-arm64", + "linux-x64-musl": "@ashbyhq/libpg-query-native-linux-x64-musl", + "linux-arm64-musl": "@ashbyhq/libpg-query-native-linux-arm64-musl", + }; + + const pkg = packageMap[platformKey]; + if (!pkg) { + throw new Error( + `Unsupported platform: ${platformKey}. ` + + `Supported: ${Object.keys(packageMap).join(", ")}` + ); + } + + try { + return require(pkg); + } catch { + throw new Error( + `Native addon not found for ${platformKey}. Install ${pkg} or ensure it's in your dependencies.` + ); + } +} + +function isMusl(): boolean { + if (process.platform !== "linux") return false; + try { + const { execSync } = require("child_process"); + const ldd = execSync("ldd --version 2>&1", { encoding: "utf8" }); + return ldd.includes("musl"); + } catch { + // ldd --version exits non-zero on musl + return true; + } +} + +interface NativeResult { + error: string | null; + result: T | null; +} + +interface NativeAddon { + parseSync(query: string): NativeResult; + parsePlPgSQLSync(query: string): NativeResult; + fingerprintSync(query: string): NativeResult; + normalizeSync(query: string): NativeResult; + scanSync(query: string): NativeResult; +} + +const addon = loadNativeAddon(); + +function checkError(res: NativeResult): void { + if (res.error) { + const details: SqlErrorDetails = JSON.parse(res.error); + throw new SqlError(details.message, details); + } +} + +export function parseSync(query: string): ParseResult { + const res = addon.parseSync(query); + checkError(res); + return JSON.parse(res.result as string); +} + +export async function parse(query: string): Promise { + return parseSync(query); +} + +export function parsePlPgSQLSync(query: string): any { + const res = addon.parsePlPgSQLSync(query); + checkError(res); + return JSON.parse(res.result as string); +} + +export async function parsePlPgSQL(query: string): Promise { + return parsePlPgSQLSync(query); +} + +export function fingerprintSync(query: string): string { + const res = addon.fingerprintSync(query); + checkError(res); + return res.result as string; +} + +export async function fingerprint(query: string): Promise { + return fingerprintSync(query); +} + +export function normalizeSync(query: string): string { + const res = addon.normalizeSync(query); + checkError(res); + return res.result as string; +} + +export async function normalize(query: string): Promise { + return normalizeSync(query); +} + +export function scanSync(query: string): ScanResult { + const res = addon.scanSync(query); + checkError(res); + return res.result as ScanResult; +} + +export async function scan(query: string): Promise { + return scanSync(query); +} + +export async function loadModule(): Promise { + // no-op for native — module is loaded synchronously via require() +} diff --git a/native/test/errors.test.js b/native/test/errors.test.js new file mode 100644 index 0000000..dfc610a --- /dev/null +++ b/native/test/errors.test.js @@ -0,0 +1,122 @@ +const { parseSync, hasSqlDetails } = require("../dist/index.js"); +const { describe, it } = require("node:test"); +const assert = require("node:assert/strict"); + +describe("Error Handling", () => { + describe("Error Details Structure", () => { + it("should include sqlDetails property on parse errors", () => { + try { + parseSync("SELECT * FROM users WHERE id = @"); + } catch (error) { + assert.ok("sqlDetails" in error); + assert.ok("message" in error.sqlDetails); + assert.ok("cursorPosition" in error.sqlDetails); + } + }); + + it("should have correct cursor position (0-based)", () => { + try { + parseSync("SELECT * FROM users WHERE id = @"); + assert.fail("Expected error"); + } catch (error) { + assert.equal(error.sqlDetails.cursorPosition, 32); + } + }); + + it("should identify error source file", () => { + try { + parseSync("SELECT * FROM users WHERE id = @"); + assert.fail("Expected error"); + } catch (error) { + assert.equal(error.sqlDetails.fileName, "scan.l"); + assert.equal(error.sqlDetails.functionName, "scanner_yyerror"); + } + }); + }); + + describe("Error Position Accuracy", () => { + const positionTests = [ + { query: "@ SELECT * FROM users", expectedPos: 0, desc: "error at start" }, + { query: "SELECT @ FROM users", expectedPos: 9, desc: "error after SELECT" }, + { query: "SELECT * FROM users WHERE id = @", expectedPos: 32, desc: "error at end" }, + ]; + + positionTests.forEach(({ query, expectedPos, desc }) => { + it(`should correctly identify position for ${desc}`, () => { + try { + parseSync(query); + assert.fail("Expected error"); + } catch (error) { + assert.equal(error.sqlDetails.cursorPosition, expectedPos); + } + }); + }); + }); + + describe("Error Types", () => { + it("should handle unterminated string literals", () => { + try { + parseSync("SELECT * FROM users WHERE name = 'unclosed"); + assert.fail("Expected error"); + } catch (error) { + assert.ok(error.message.includes("unterminated quoted string")); + } + }); + + it("should handle reserved keywords", () => { + try { + parseSync("SELECT * FROM table"); + assert.fail("Expected error"); + } catch (error) { + assert.ok(error.message.includes('syntax error at or near "table"')); + } + }); + }); + + describe("Edge Cases", () => { + it("should handle empty query", () => { + assert.throws(() => parseSync(""), { + message: "Query cannot be empty", + }); + }); + + it("should handle @ in strings", () => { + const query = "SELECT * FROM users WHERE email = 'user@example.com'"; + assert.doesNotThrow(() => parseSync(query)); + }); + }); + + describe("hasSqlDetails Type Guard", () => { + it("should return true for SQL parse errors", () => { + try { + parseSync("SELECT * FROM users WHERE id = @"); + assert.fail("Expected error"); + } catch (error) { + assert.equal(hasSqlDetails(error), true); + } + }); + + it("should return false for regular errors", () => { + assert.equal(hasSqlDetails(new Error("Regular error")), false); + }); + + it("should return false for non-Error objects", () => { + assert.equal(hasSqlDetails("string"), false); + assert.equal(hasSqlDetails(null), false); + assert.equal(hasSqlDetails(undefined), false); + }); + }); + + describe("Backward Compatibility", () => { + it("should maintain Error instance", () => { + try { + parseSync("SELECT * FROM users WHERE id = @"); + assert.fail("Expected error"); + } catch (error) { + assert.ok(error instanceof Error); + assert.ok(error.message); + assert.ok(error.stack); + } + }); + }); +}); diff --git a/native/test/fingerprint.test.js b/native/test/fingerprint.test.js new file mode 100644 index 0000000..cb3afcb --- /dev/null +++ b/native/test/fingerprint.test.js @@ -0,0 +1,57 @@ +const query = require("../dist/index.js"); +const { describe, it } = require("node:test"); +const assert = require("node:assert/strict"); + +describe("Query Fingerprinting", () => { + describe("Sync Fingerprinting", () => { + it("should return a fingerprint for a simple query", () => { + const fingerprint = query.fingerprintSync("select 1"); + assert.equal(typeof fingerprint, "string"); + assert.equal(fingerprint.length, 16); + }); + + it("should return same fingerprint for equivalent queries", () => { + const fp1 = query.fingerprintSync("select 1"); + const fp2 = query.fingerprintSync("SELECT 1"); + const fp3 = query.fingerprintSync("select 1"); + assert.equal(fp1, fp2); + assert.equal(fp1, fp3); + }); + + it("should return different fingerprints for different queries", () => { + const fp1 = query.fingerprintSync("select name from users"); + const fp2 = query.fingerprintSync("select id from customers"); + assert.notEqual(fp1, fp2); + }); + + it("should normalize parameter values", () => { + const fp1 = query.fingerprintSync("select * from users where id = 123"); + const fp2 = query.fingerprintSync("select * from users where id = 456"); + assert.equal(fp1, fp2); + }); + + it("should fail on invalid queries", () => { + assert.throws(() => query.fingerprintSync("NOT A QUERY"), Error); + }); + }); + + describe("Async Fingerprinting", () => { + it("should return a promise resolving to same result", async () => { + const testQuery = "select * from users"; + const fp = await query.fingerprint(testQuery); + assert.equal(fp, query.fingerprintSync(testQuery)); + }); + + it("should reject on bogus queries", async () => { + return query.fingerprint("NOT A QUERY").then( + () => { + throw new Error("should have rejected"); + }, + (e) => { + assert.ok(e instanceof Error); + assert.match(e.message, /NOT/); + } + ); + }); + }); +}); diff --git a/native/test/normalize.test.js b/native/test/normalize.test.js new file mode 100644 index 0000000..6f66584 --- /dev/null +++ b/native/test/normalize.test.js @@ -0,0 +1,61 @@ +const query = require("../dist/index.js"); +const { describe, it } = require("node:test"); +const assert = require("node:assert/strict"); + +describe("Query Normalization", () => { + describe("Sync Normalization", () => { + it("should normalize a simple query", () => { + const normalized = query.normalizeSync("select 1"); + assert.equal(typeof normalized, "string"); + assert.ok(normalized.includes("$1")); + }); + + it("should normalize parameter values", () => { + const n1 = query.normalizeSync("select * from users where id = 123"); + const n2 = query.normalizeSync("select * from users where id = 456"); + assert.equal(n1, n2); + assert.ok(n1.includes("$1")); + }); + + it("should normalize string literals", () => { + const n1 = query.normalizeSync("select * from users where name = 'john'"); + const n2 = query.normalizeSync("select * from users where name = 'jane'"); + assert.equal(n1, n2); + assert.ok(n1.includes("$1")); + }); + + it("should preserve query structure", () => { + const normalized = query.normalizeSync( + "SELECT id, name FROM users WHERE active = true ORDER BY name" + ); + assert.ok(normalized.includes("SELECT")); + assert.ok(normalized.includes("FROM")); + assert.ok(normalized.includes("WHERE")); + assert.ok(normalized.includes("ORDER BY")); + }); + + it("should fail on invalid queries", () => { + assert.throws(() => query.normalizeSync("NOT A QUERY"), Error); + }); + }); + + describe("Async Normalization", () => { + it("should return a promise resolving to same result", async () => { + const testQuery = "select * from users where id = 123"; + const normalized = await query.normalize(testQuery); + assert.equal(normalized, query.normalizeSync(testQuery)); + }); + + it("should reject on bogus queries", async () => { + return query.normalize("NOT A QUERY").then( + () => { + throw new Error("should have rejected"); + }, + (e) => { + assert.ok(e instanceof Error); + assert.match(e.message, /NOT/); + } + ); + }); + }); +}); diff --git a/native/test/parsing.test.js b/native/test/parsing.test.js new file mode 100644 index 0000000..e8d3b7c --- /dev/null +++ b/native/test/parsing.test.js @@ -0,0 +1,76 @@ +const query = require("../dist/index.js"); +const { describe, it } = require("node:test"); +const assert = require("node:assert/strict"); + +function removeLocationProperties(obj) { + if (typeof obj !== "object" || obj === null) return obj; + if (Array.isArray(obj)) return obj.map(removeLocationProperties); + const result = {}; + for (const key in obj) { + if (key === "location" || key === "stmt_len" || key === "stmt_location") + continue; + result[key] = removeLocationProperties(obj[key]); + } + return result; +} + +describe("Query Parsing", () => { + describe("Sync Parsing", () => { + it("should return a single-item parse result for common queries", () => { + const queries = ["select 1", "select null", "select ''", "select a, b"]; + const results = queries.map(query.parseSync); + results.forEach((res) => { + assert.equal(res.stmts.length, 1); + }); + + const selectedDatas = results.map( + (it) => it.stmts[0].stmt.SelectStmt.targetList + ); + + assert.equal(selectedDatas[0][0].ResTarget.val.A_Const.ival.ival, 1); + assert.equal(selectedDatas[1][0].ResTarget.val.A_Const.isnull, true); + assert.equal(selectedDatas[2][0].ResTarget.val.A_Const.sval.sval, ""); + assert.equal(selectedDatas[3].length, 2); + }); + + it("should support parsing multiple queries", () => { + const res = query.parseSync("select 1; select null;"); + assert.deepEqual( + res.stmts.map(removeLocationProperties), + [ + ...query.parseSync("select 1;").stmts.map(removeLocationProperties), + ...query + .parseSync("select null;") + .stmts.map(removeLocationProperties), + ] + ); + }); + + it("should not parse a bogus query", () => { + assert.throws(() => query.parseSync("NOT A QUERY"), Error); + }); + }); + + describe("Async parsing", () => { + it("should return a promise resolving to same result", async () => { + const testQuery = "select * from john;"; + const resPromise = query.parse(testQuery); + const res = await resPromise; + + assert.ok(resPromise instanceof Promise); + assert.deepEqual(res, query.parseSync(testQuery)); + }); + + it("should reject on bogus queries", async () => { + return query.parse("NOT A QUERY").then( + () => { + throw new Error("should have rejected"); + }, + (e) => { + assert.ok(e instanceof Error); + assert.match(e.message, /NOT/); + } + ); + }); + }); +}); diff --git a/native/test/plpgsql.test.js b/native/test/plpgsql.test.js new file mode 100644 index 0000000..5a4a81b --- /dev/null +++ b/native/test/plpgsql.test.js @@ -0,0 +1,65 @@ +const query = require("../dist/index.js"); +const { describe, it } = require("node:test"); +const assert = require("node:assert/strict"); + +describe("PL/pgSQL Parsing", () => { + describe("Sync PL/pgSQL Parsing", () => { + it("should parse a simple PL/pgSQL function", () => { + const funcSql = ` + CREATE OR REPLACE FUNCTION test_func() + RETURNS INTEGER AS $$ + BEGIN + RETURN 42; + END; + $$ LANGUAGE plpgsql; + `; + const result = query.parsePlPgSQLSync(funcSql); + assert.equal(typeof result, "object"); + assert.ok(Array.isArray(result.plpgsql_funcs)); + }); + + it("should parse function with parameters", () => { + const funcSql = ` + CREATE OR REPLACE FUNCTION add_numbers(a INTEGER, b INTEGER) + RETURNS INTEGER AS $$ + BEGIN + RETURN a + b; + END; + $$ LANGUAGE plpgsql; + `; + const result = query.parsePlPgSQLSync(funcSql); + assert.ok(result.plpgsql_funcs.length > 0); + }); + + it("should fail on invalid PL/pgSQL", () => { + assert.throws(() => query.parsePlPgSQLSync("NOT A FUNCTION"), Error); + }); + }); + + describe("Async PL/pgSQL Parsing", () => { + it("should return a promise resolving to same result", async () => { + const funcSql = ` + CREATE OR REPLACE FUNCTION test_func() + RETURNS INTEGER AS $$ + BEGIN + RETURN 42; + END; + $$ LANGUAGE plpgsql; + `; + const result = await query.parsePlPgSQL(funcSql); + assert.deepEqual(result, query.parsePlPgSQLSync(funcSql)); + }); + + it("should reject on invalid PL/pgSQL", async () => { + return query.parsePlPgSQL("NOT A FUNCTION").then( + () => { + throw new Error("should have rejected"); + }, + (e) => { + assert.ok(e instanceof Error); + assert.match(e.message, /NOT/); + } + ); + }); + }); +}); diff --git a/native/test/scan.test.js b/native/test/scan.test.js new file mode 100644 index 0000000..7143f30 --- /dev/null +++ b/native/test/scan.test.js @@ -0,0 +1,88 @@ +const query = require("../dist/index.js"); +const { describe, it } = require("node:test"); +const assert = require("node:assert/strict"); + +describe("Query Scanning", () => { + describe("Sync Scanning", () => { + it("should return a scan result with version and tokens", () => { + const result = query.scanSync("SELECT 1"); + assert.equal(typeof result, "object"); + assert.ok("version" in result); + assert.ok("tokens" in result); + assert.ok(Array.isArray(result.tokens)); + }); + + it("should scan a simple SELECT query correctly", () => { + const result = query.scanSync("SELECT 1"); + assert.equal(result.tokens.length, 2); + + const selectToken = result.tokens[0]; + assert.equal(selectToken.text, "SELECT"); + assert.equal(selectToken.start, 0); + assert.equal(selectToken.end, 6); + assert.equal(selectToken.keywordName, "RESERVED_KEYWORD"); + + const numberToken = result.tokens[1]; + assert.equal(numberToken.text, "1"); + assert.equal(numberToken.start, 7); + assert.equal(numberToken.end, 8); + assert.equal(numberToken.tokenName, "ICONST"); + assert.equal(numberToken.keywordName, "NO_KEYWORD"); + }); + + it("should scan tokens with correct positions", () => { + const sql = "SELECT * FROM users"; + const result = query.scanSync(sql); + assert.equal(result.tokens.length, 4); + result.tokens.forEach((token) => { + const actualText = sql.substring(token.start, token.end); + assert.equal(token.text, actualText); + }); + }); + + it("should identify different token types", () => { + const result = query.scanSync( + "SELECT 'string', 123, 3.14, $1 FROM users" + ); + const tokenTypes = result.tokens.map((t) => t.tokenName); + assert.ok(tokenTypes.includes("SCONST")); + assert.ok(tokenTypes.includes("ICONST")); + assert.ok(tokenTypes.includes("FCONST")); + assert.ok(tokenTypes.includes("PARAM")); + }); + + it("should handle complex queries with parameters", () => { + const result = query.scanSync( + "SELECT * FROM users WHERE id = $1 AND name = $2" + ); + const params = result.tokens.filter((t) => t.tokenName === "PARAM"); + assert.equal(params.length, 2); + assert.equal(params[0].text, "$1"); + assert.equal(params[1].text, "$2"); + }); + + it("should handle special PostgreSQL operators", () => { + const result = query.scanSync("SELECT id::text FROM users"); + const typecast = result.tokens.find((t) => t.text === "::"); + assert.ok(typecast); + assert.equal(typecast.tokenName, "TYPECAST"); + }); + + it("should preserve original token order", () => { + const result = query.scanSync( + "SELECT id, name FROM users ORDER BY name" + ); + for (let i = 1; i < result.tokens.length; i++) { + assert.ok(result.tokens[i].start >= result.tokens[i - 1].end); + } + }); + }); + + describe("Async Scanning", () => { + it("should return a promise resolving to same result as sync", async () => { + const testQuery = "SELECT * FROM users WHERE id = $1"; + const result = await query.scan(testQuery); + assert.deepEqual(result, query.scanSync(testQuery)); + }); + }); +}); diff --git a/native/tsconfig.json b/native/tsconfig.json new file mode 100644 index 0000000..3f3bd4a --- /dev/null +++ b/native/tsconfig.json @@ -0,0 +1,15 @@ +{ + "compilerOptions": { + "target": "ES2020", + "module": "commonjs", + "moduleResolution": "node", + "declaration": true, + "outDir": "dist", + "rootDir": "src", + "strict": true, + "esModuleInterop": true, + "skipLibCheck": true + }, + "include": ["src/**/*.ts"], + "exclude": ["node_modules", "dist", "test", "benchmark"] +} diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml index 7135354..2eae5cc 100644 --- a/pnpm-workspace.yaml +++ b/pnpm-workspace.yaml @@ -1,6 +1,7 @@ packages: - 'parser' - 'full' + - 'native' - 'versions/18' - 'versions/17' - 'versions/16' From 1c6d3c0708b82383a36ddd9204e9db4b4aa6a6a4 Mon Sep 17 00:00:00 2001 From: Ben Asher Date: Sun, 28 Jun 2026 14:28:10 -0700 Subject: [PATCH 02/20] Add multi-allocator comparison benchmark Tests system malloc, jemalloc, mimalloc, and tcmalloc via LD_PRELOAD. Supports multiple parse/free cycles (--cycles N) to reveal ratchet behavior, and auto-detects which allocator is active from environment variables. Co-Authored-By: Claude Opus 4.6 --- native/benchmark/compare-allocators.sh | 94 +++++++++++++++++++++ native/benchmark/memory.mjs | 109 ++++++++++++++++--------- 2 files changed, 164 insertions(+), 39 deletions(-) create mode 100755 native/benchmark/compare-allocators.sh diff --git a/native/benchmark/compare-allocators.sh b/native/benchmark/compare-allocators.sh new file mode 100755 index 0000000..c96dc58 --- /dev/null +++ b/native/benchmark/compare-allocators.sh @@ -0,0 +1,94 @@ +#!/bin/bash +# Compare allocator behavior for the native libpg-query addon. +# +# Tests: system malloc, jemalloc, mimalloc, tcmalloc +# Measures: idle RSS, peak RSS, post-free RSS, throughput +# +# Usage: +# bash benchmark/compare-allocators.sh # large query (default) +# bash benchmark/compare-allocators.sh --small # small query sanity check +# bash benchmark/compare-allocators.sh --throughput # include throughput numbers + +set -e + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +BENCHMARK="$SCRIPT_DIR/memory.mjs" +ARGS=("$@") + +# Detect allocator libraries +JEMALLOC_LIB="" +MIMALLOC_LIB="" +TCMALLOC_LIB="" + +if [[ "$(uname)" == "Darwin" ]]; then + JEMALLOC_LIB="$(brew --prefix jemalloc 2>/dev/null)/lib/libjemalloc.dylib" + MIMALLOC_LIB="$(brew --prefix mimalloc 2>/dev/null)/lib/libmimalloc.dylib" + TCMALLOC_LIB="$(brew --prefix gperftools 2>/dev/null)/lib/libtcmalloc.dylib" + PRELOAD_VAR="DYLD_INSERT_LIBRARIES" +else + for candidate in \ + /usr/lib/x86_64-linux-gnu/libjemalloc.so.2 \ + /usr/lib/aarch64-linux-gnu/libjemalloc.so.2 \ + /usr/lib64/libjemalloc.so.2 \ + /usr/lib/libjemalloc.so; do + [[ -f "$candidate" ]] && JEMALLOC_LIB="$candidate" && break + done + for candidate in \ + /usr/lib/x86_64-linux-gnu/libmimalloc.so \ + /usr/lib/aarch64-linux-gnu/libmimalloc.so \ + /usr/lib64/libmimalloc.so \ + /usr/lib/libmimalloc.so; do + [[ -f "$candidate" ]] && MIMALLOC_LIB="$candidate" && break + done + for candidate in \ + /usr/lib/x86_64-linux-gnu/libtcmalloc.so \ + /usr/lib/aarch64-linux-gnu/libtcmalloc.so \ + /usr/lib64/libtcmalloc.so \ + /usr/lib/libtcmalloc.so; do + [[ -f "$candidate" ]] && TCMALLOC_LIB="$candidate" && break + done + PRELOAD_VAR="LD_PRELOAD" +fi + +echo "=============================================" +echo " Allocator Comparison Benchmark" +echo " $(uname -s) $(uname -m), Node $(node -v)" +echo "=============================================" +echo "" +echo "Allocators found:" +echo " system malloc: yes (always available)" +[[ -f "$JEMALLOC_LIB" ]] && echo " jemalloc: $JEMALLOC_LIB" || echo " jemalloc: not found" +[[ -f "$MIMALLOC_LIB" ]] && echo " mimalloc: $MIMALLOC_LIB" || echo " mimalloc: not found" +[[ -f "$TCMALLOC_LIB" ]] && echo " tcmalloc: $TCMALLOC_LIB" || echo " tcmalloc: not found" +echo "" + +run_benchmark() { + local name="$1" + local preload="$2" + + echo ">>> $name" + if [[ -n "$preload" ]]; then + env "$PRELOAD_VAR=$preload" node --expose-gc "$BENCHMARK" "${ARGS[@]}" 2>&1 + else + node --expose-gc "$BENCHMARK" "${ARGS[@]}" 2>&1 + fi + echo "" +} + +run_benchmark "System malloc (default)" "" + +[[ -f "$JEMALLOC_LIB" ]] && run_benchmark "jemalloc" "$JEMALLOC_LIB" +[[ -f "$MIMALLOC_LIB" ]] && run_benchmark "mimalloc" "$MIMALLOC_LIB" +[[ -f "$TCMALLOC_LIB" ]] && run_benchmark "tcmalloc" "$TCMALLOC_LIB" + +# jemalloc with aggressive decay +if [[ -f "$JEMALLOC_LIB" ]]; then + echo ">>> jemalloc (aggressive decay: dirty=0, muzzy=0)" + env "$PRELOAD_VAR=$JEMALLOC_LIB" MALLOC_CONF="dirty_decay_ms:0,muzzy_decay_ms:0" \ + node --expose-gc "$BENCHMARK" "${ARGS[@]}" 2>&1 + echo "" +fi + +echo "=============================================" +echo " Done. Compare Peak RSS and Retained columns." +echo "=============================================" diff --git a/native/benchmark/memory.mjs b/native/benchmark/memory.mjs index 2b75a37..72d7d13 100644 --- a/native/benchmark/memory.mjs +++ b/native/benchmark/memory.mjs @@ -1,16 +1,16 @@ /** * Memory benchmark: compare allocator behavior for large SQL parses. * - * Generates a ~2.74 MB SQL query (1500× UNION ALL of SELECT c0..c399 FROM t), + * Generates a ~3.3 MB SQL query (1500× UNION ALL of SELECT c0..c399 FROM t), * parses it, and measures RSS at idle, peak (post-parse), and after free. * * Usage: - * node benchmark/memory.mjs # native (jemalloc) + * node benchmark/memory.mjs # native backend * node benchmark/memory.mjs --wasm # WASM backend for comparison - * node benchmark/memory.mjs --system-malloc # native without jemalloc * node benchmark/memory.mjs --all # run all available backends * node benchmark/memory.mjs --small # quick sanity check (small query) * node benchmark/memory.mjs --throughput # throughput benchmark + * node benchmark/memory.mjs --cycles 3 # multiple parse/free cycles */ import { argv } from "process"; @@ -21,10 +21,6 @@ function rss() { return process.memoryUsage.rss(); } -function rssMB() { - return (rss() / MB).toFixed(1); -} - function generateBigQuery(unions = 1500, cols = 400) { const colList = Array.from({ length: cols }, (_, i) => `c${i}`).join(", "); const select = `SELECT ${colList} FROM t`; @@ -35,41 +31,74 @@ function generateSmallQuery() { return "SELECT id, name, email, created_at FROM users WHERE id = $1 AND active = true"; } -async function benchmarkParse(backend, label, query) { - // Force GC if available +function detectAllocator() { + const preload = process.env.LD_PRELOAD || process.env.DYLD_INSERT_LIBRARIES || ""; + if (preload.includes("jemalloc")) return "jemalloc"; + if (preload.includes("mimalloc")) return "mimalloc"; + if (preload.includes("tcmalloc")) return "tcmalloc"; + return "system"; +} + +async function gcAndSettle(ms = 200) { if (global.gc) global.gc(); - await new Promise((r) => setTimeout(r, 100)); + await new Promise((r) => setTimeout(r, ms)); + if (global.gc) global.gc(); + await new Promise((r) => setTimeout(r, ms)); +} + +async function benchmarkParse(backend, label, query, cycles = 1) { + await gcAndSettle(); const idleRss = rss(); console.log(`\n--- ${label} ---`); console.log(`Query size: ${(query.length / MB).toFixed(2)} MB`); console.log(`Idle RSS: ${(idleRss / MB).toFixed(1)} MB`); - const start = performance.now(); - let result; - try { - result = await backend.parse(query); - } catch (e) { - console.log(`Parse failed: ${e.message}`); - return null; + let peakRss = 0; + let totalParseTime = 0; + let resultSize = 0; + + for (let cycle = 0; cycle < cycles; cycle++) { + if (cycles > 1) process.stdout.write(` cycle ${cycle + 1}/${cycles}...`); + + const start = performance.now(); + let result; + try { + result = await backend.parse(query); + } catch (e) { + console.log(` parse failed: ${e.message}`); + return null; + } + const elapsed = performance.now() - start; + totalParseTime += elapsed; + + const currentRss = rss(); + if (currentRss > peakRss) peakRss = currentRss; + + if (cycle === 0) { + const resultJson = JSON.stringify(result); + resultSize = Buffer.byteLength(resultJson); + } + + // Drop references and let GC + allocator return memory + result = null; + await gcAndSettle(300); + + const afterCycleRss = rss(); + if (cycles > 1) { + console.log( + ` peak=${(currentRss / MB).toFixed(0)} MB, after=${(afterCycleRss / MB).toFixed(0)} MB, ${elapsed.toFixed(0)} ms` + ); + } } - const parseTime = performance.now() - start; - const peakRss = rss(); - const resultJson = JSON.stringify(result); - const resultSize = Buffer.byteLength(resultJson); + const afterRss = rss(); + const avgParseTime = totalParseTime / cycles; - console.log(`Parse time: ${parseTime.toFixed(0)} ms`); + console.log(`Parse time: ${avgParseTime.toFixed(0)} ms${cycles > 1 ? ` (avg of ${cycles})` : ""}`); console.log(`Result size: ${(resultSize / MB).toFixed(1)} MB`); console.log(`Peak RSS: ${(peakRss / MB).toFixed(1)} MB`); console.log(`Peak delta: +${((peakRss - idleRss) / MB).toFixed(1)} MB`); - - // Drop references and let GC + allocator return memory - result = null; - if (global.gc) global.gc(); - await new Promise((r) => setTimeout(r, 500)); - - const afterRss = rss(); console.log(`After free: ${(afterRss / MB).toFixed(1)} MB`); console.log(`Retained: +${((afterRss - idleRss) / MB).toFixed(1)} MB`); @@ -78,7 +107,7 @@ async function benchmarkParse(backend, label, query) { idleRss, peakRss, afterRss, - parseTime, + parseTime: avgParseTime, resultSize, querySize: query.length, }; @@ -130,11 +159,6 @@ async function loadWasm() { } } -function isJemallocLoaded() { - return !!(process.env.LD_PRELOAD?.includes("jemalloc") || - process.env.DYLD_INSERT_LIBRARIES?.includes("jemalloc")); -} - async function main() { const flags = new Set(argv.slice(2)); @@ -144,10 +168,18 @@ async function main() { const small = flags.has("--small"); const throughput = flags.has("--throughput"); + let cycles = 1; + const cyclesIdx = argv.indexOf("--cycles"); + if (cyclesIdx !== -1 && argv[cyclesIdx + 1]) { + cycles = parseInt(argv[cyclesIdx + 1], 10) || 1; + } + const query = small ? generateSmallQuery() : generateBigQuery(); + const allocator = detectAllocator(); console.log("=== libpg-query Memory Benchmark ==="); console.log(`Node ${process.version}, ${process.platform}-${process.arch}`); + console.log(`Allocator: ${allocator}`); console.log(`GC exposed: ${typeof global.gc === "function"}`); if (typeof global.gc !== "function") { console.log("Hint: run with --expose-gc for more accurate measurements"); @@ -158,9 +190,8 @@ async function main() { if (runNative) { const native = await loadNative(); if (native) { - const jemallocLoaded = isJemallocLoaded(); - const label = jemallocLoaded ? "Native (jemalloc)" : "Native (system malloc)"; - results.push(await benchmarkParse(native, label, query)); + const label = `Native (${allocator})`; + results.push(await benchmarkParse(native, label, query, cycles)); if (throughput) { await benchmarkThroughput(native, label); } @@ -170,7 +201,7 @@ async function main() { if (runWasm) { const wasm = await loadWasm(); if (wasm) { - results.push(await benchmarkParse(wasm, "WASM (emscripten)", query)); + results.push(await benchmarkParse(wasm, "WASM (emscripten)", query, cycles)); if (throughput) { await benchmarkThroughput(wasm, "WASM (emscripten)"); } From 326e25d542c77b1d23ae6e1e1b0476b398b31066 Mon Sep 17 00:00:00 2001 From: Ben Asher Date: Sun, 28 Jun 2026 16:28:17 -0700 Subject: [PATCH 03/20] Add CI/CD, cross-platform builds, and package metadata for native backend Builds on the native N-API backend with the release/distribution plumbing: - platforms.json as the single source of truth for the 6 target platforms (os/cpu/libc + CI runner/container). package-platforms.mjs, the runtime loader in index.ts, and sync-optional-deps.mjs all read from it, so adding a platform is a one-line change. - Mark glibc vs musl Linux packages mutually exclusive via `libc`, so npm 10+ and Yarn Berry install only the binary matching the host. - native-build.yml: reusable workflow that derives its matrix from platforms.json, builds + tests on every platform, uploads prebuilds. - native-ci.yml: runs the build, then a package smoke test that installs the generated platform package in an isolated dir and exercises every public API. - native-release.yml: manual dispatch (with dry-run) that builds all platforms and publishes the main + per-platform packages to npm. - README: replace prior unverified memory figures with reproducible, methodology-labeled system-malloc-vs-jemalloc measurements. Co-Authored-By: Claude Opus 4.8 --- .github/workflows/native-build.yml | 75 +++++++++++++++++ .github/workflows/native-ci.yml | 93 +++++++++++++++++++++ .github/workflows/native-release.yml | 115 ++++++++++++++++++++++++++ native/.gitignore | 1 + native/README.md | 90 +++++++++++++++----- native/benchmark/breakdown.mjs | 101 ++++++++++++++++++++++ native/package.json | 3 +- native/platforms.json | 38 +++++++++ native/scripts/package-platforms.mjs | 16 ++-- native/scripts/sync-optional-deps.mjs | 21 +++++ native/src/index.ts | 21 ++--- native/test/smoke.mjs | 100 ++++++++++++++++++++++ 12 files changed, 626 insertions(+), 48 deletions(-) create mode 100644 .github/workflows/native-build.yml create mode 100644 .github/workflows/native-ci.yml create mode 100644 .github/workflows/native-release.yml create mode 100644 native/benchmark/breakdown.mjs create mode 100644 native/platforms.json create mode 100644 native/scripts/sync-optional-deps.mjs create mode 100644 native/test/smoke.mjs diff --git a/.github/workflows/native-build.yml b/.github/workflows/native-build.yml new file mode 100644 index 0000000..d2fc9bd --- /dev/null +++ b/.github/workflows/native-build.yml @@ -0,0 +1,75 @@ +name: Native Build (reusable) + +on: + workflow_call: + inputs: + upload-retention-days: + type: number + default: 7 + +jobs: + matrix: + name: Generate matrix + runs-on: ubuntu-24.04 + outputs: + include: ${{ steps.gen.outputs.include }} + steps: + - uses: actions/checkout@v4 + - id: gen + run: | + include=$(node -e " + const p = require('./native/platforms.json'); + const matrix = Object.entries(p).map(([key, v]) => ({ + platform: key, + runner: v.runner, + container: v.container || '', + musl: !!v.libc, + })); + console.log(JSON.stringify(matrix)); + ") + echo "include=${include}" >> "$GITHUB_OUTPUT" + + build: + name: Build ${{ matrix.platform }} + needs: matrix + runs-on: ${{ matrix.runner }} + strategy: + fail-fast: false + matrix: + include: ${{ fromJSON(needs.matrix.outputs.include) }} + container: ${{ matrix.container || '' }} + steps: + - name: Install Alpine build deps + if: matrix.musl + run: apk add --no-cache git make g++ python3 + + - uses: actions/checkout@v4 + + - name: Setup Node.js + if: "!matrix.musl" + uses: actions/setup-node@v4 + with: + node-version: "20" + + - name: Install dependencies + working-directory: native + run: npm install + + - name: Build native addon + working-directory: native + run: make build + + - name: Build TypeScript + working-directory: native + run: npx tsc + + - name: Run tests + working-directory: native + run: npm test + + - name: Upload prebuild + uses: actions/upload-artifact@v4 + with: + name: prebuild-${{ matrix.platform }} + path: native/prebuilds/${{ matrix.platform }}/libpg_query_native.node + retention-days: ${{ inputs.upload-retention-days }} diff --git a/.github/workflows/native-ci.yml b/.github/workflows/native-ci.yml new file mode 100644 index 0000000..81a09a1 --- /dev/null +++ b/.github/workflows/native-ci.yml @@ -0,0 +1,93 @@ +name: Native CI + +on: + push: + branches: [napi-jemalloc, main] + paths: + - "native/**" + - ".github/workflows/native-ci.yml" + - ".github/workflows/native-build.yml" + pull_request: + paths: + - "native/**" + - ".github/workflows/native-ci.yml" + - ".github/workflows/native-build.yml" + +jobs: + build: + name: Build + uses: ./.github/workflows/native-build.yml + with: + upload-retention-days: 3 + + package-test-matrix: + name: Generate package test matrix + runs-on: ubuntu-24.04 + outputs: + include: ${{ steps.gen.outputs.include }} + steps: + - uses: actions/checkout@v4 + - id: gen + run: | + include=$(node -e " + const p = require('./native/platforms.json'); + const matrix = Object.entries(p).map(([key, v]) => ({ + platform: key, + runner: v.runner, + container: v.container || '', + musl: !!v.libc, + })); + console.log(JSON.stringify(matrix)); + ") + echo "include=${include}" >> "$GITHUB_OUTPUT" + + package-test: + name: Package test ${{ matrix.platform }} + needs: [build, package-test-matrix] + runs-on: ${{ matrix.runner }} + strategy: + fail-fast: false + matrix: + include: ${{ fromJSON(needs.package-test-matrix.outputs.include) }} + container: ${{ matrix.container || '' }} + steps: + - name: Install Alpine deps + if: matrix.musl + run: apk add --no-cache git + + - uses: actions/checkout@v4 + + - name: Setup Node.js + if: "!matrix.musl" + uses: actions/setup-node@v4 + with: + node-version: "20" + + - name: Download prebuild + uses: actions/download-artifact@v4 + with: + name: prebuild-${{ matrix.platform }} + path: native/prebuilds/${{ matrix.platform }} + + - name: Install, build TS, and generate platform package + working-directory: native + run: | + npm install + npx tsc + node scripts/package-platforms.mjs + + - name: Install package in isolated directory + run: | + mkdir -p /tmp/smoke-test + cd /tmp/smoke-test + npm init -y + npm install "$GITHUB_WORKSPACE/native" --install-strategy=nested + # Install the platform package from generated dir + platform_pkg=$(ls -d "$GITHUB_WORKSPACE/native/packages/"*${{ matrix.platform }}*) + npm install "$platform_pkg" --install-strategy=nested + + - name: Run smoke test + run: | + cd /tmp/smoke-test + cp "$GITHUB_WORKSPACE/native/test/smoke.mjs" smoke.mjs + node smoke.mjs diff --git a/.github/workflows/native-release.yml b/.github/workflows/native-release.yml new file mode 100644 index 0000000..5944088 --- /dev/null +++ b/.github/workflows/native-release.yml @@ -0,0 +1,115 @@ +name: Native Release + +on: + workflow_dispatch: + inputs: + version: + description: "Version to publish (e.g. 0.1.0)" + required: true + dry-run: + description: "Dry run (skip npm publish)" + type: boolean + default: true + +jobs: + build: + name: Build + uses: ./.github/workflows/native-build.yml + with: + upload-retention-days: 1 + + publish: + name: Publish to npm + needs: build + runs-on: ubuntu-24.04 + permissions: + contents: write + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-node@v4 + with: + node-version: "20" + registry-url: "https://registry.npmjs.org" + + - name: Download all prebuilds + uses: actions/download-artifact@v4 + with: + path: native/prebuilds + pattern: prebuild-* + merge-multiple: false + + - name: Arrange prebuilds into expected layout + working-directory: native + run: | + for dir in prebuilds/prebuild-*/; do + platform=$(basename "$dir" | sed 's/prebuild-//') + mkdir -p "prebuilds/${platform}" + mv "${dir}libpg_query_native.node" "prebuilds/${platform}/" + rmdir "$dir" + done + echo "Prebuilds:" + find prebuilds -name '*.node' | sort + + - name: Set version + working-directory: native + run: | + npm version "${{ inputs.version }}" --no-git-tag-version + node scripts/sync-optional-deps.mjs + echo "Version set to ${{ inputs.version }}" + + - name: Install dependencies + working-directory: native + run: npm install + + - name: Build TypeScript + working-directory: native + run: npx tsc + + - name: Generate platform packages + working-directory: native + run: node scripts/package-platforms.mjs + + - name: Publish platform packages + if: ${{ !inputs.dry-run }} + working-directory: native + env: + NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }} + run: | + for pkg_dir in packages/*/; do + echo "Publishing $(basename $pkg_dir)..." + cd "$pkg_dir" + npm publish --access public + cd ../.. + done + + - name: Publish main package + if: ${{ !inputs.dry-run }} + working-directory: native + env: + NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }} + run: npm publish --access public + + - name: Dry run summary + if: ${{ inputs.dry-run }} + working-directory: native + run: | + echo "=== DRY RUN ===" + echo "" + echo "Would publish version ${{ inputs.version }}:" + echo "" + echo "Platform packages:" + for pkg_dir in packages/*/; do + size=$(du -h "${pkg_dir}libpg_query_native.node" | cut -f1) + echo " $(basename $pkg_dir): ${size}" + done + echo "" + echo "Main package: @ashbyhq/libpg-query-native" + echo "" + echo "To publish for real, re-run with dry-run unchecked." + + - name: Create git tag + if: ${{ !inputs.dry-run }} + run: | + git tag "native-v${{ inputs.version }}" + git push origin "native-v${{ inputs.version }}" diff --git a/native/.gitignore b/native/.gitignore index 67b4883..1176e91 100644 --- a/native/.gitignore +++ b/native/.gitignore @@ -3,3 +3,4 @@ prebuilds/ packages/ dist/ node_modules/ +package-lock.json diff --git a/native/README.md b/native/README.md index f001712..54d1c34 100644 --- a/native/README.md +++ b/native/README.md @@ -4,20 +4,37 @@ Native N-API PostgreSQL query parser — a memory-efficient alternative to the W ## Why native? -The WASM build (`@libpg-query/parser`) has unavoidable memory behavior for large queries: -- Peak RSS ~935 MB for a 2.74 MB query (vs ~70 MB native with jemalloc) -- WebAssembly linear memory never shrinks — RSS ratchets up permanently -- No allocator can fix this within WASM constraints - -The native build eliminates both problems. With jemalloc preloaded, RSS tracks the live set and returns to baseline after each parse. - -``` - idle big-query peak ratchet throughput -WASM (today) 32 MB 935 MB permanent 126k/s -Native ~4 MB ~70 MB* none* 259k/s -``` - -\* With jemalloc. Without jemalloc, macOS libmalloc retains ~274 MB (still 3.5x better than WASM). +The WASM build (`@libpg-query/parser`) carries a structural memory cost: WebAssembly +linear memory only ever grows. Once a large parse expands the heap, that memory is +never returned to the OS, so a process that parses one big query keeps the high-water +mark for its lifetime, and repeated large parses ratchet RSS upward monotonically. +No allocator choice can change this — it's a property of the WASM memory model. + +The native build removes that ceiling: it uses the host allocator, so freed memory can +actually be returned to the OS. Pairing it with **jemalloc** (via `LD_PRELOAD` / +`DYLD_INSERT_LIBRARIES`) roughly halves peak RSS and, more importantly, keeps it +*stable* across repeated large parses instead of ratcheting. + +### Measured: native, system malloc vs jemalloc + +Parsing a 3.31 MB SQL query (1500× `UNION ALL`, ~65 MB JSON parse tree), 3 parse/free +cycles, `darwin-arm64`, Node 24. Peak is the max RSS during a parse; "after" is RSS +once the result is dropped and GC settles. + +| Allocator | cycle 1 peak | cycle 2 peak | cycle 3 peak | retained after | +|-----------|-------------|-------------|-------------|----------------| +| system malloc | 649 MB | 867 MB | **932 MB** | 867 MB | +| jemalloc | 381 MB | 497 MB | **498 MB** | 433 MB | +| jemalloc (`dirty_decay_ms:0,muzzy_decay_ms:0`) | 377 MB | 477 MB | **477 MB** | 411 MB | + +System malloc climbs every cycle (649 → 932 MB) — it fragments and holds the freed +arenas. jemalloc stabilizes at ~498 MB by cycle 2 and stays there. Throughput is +identical either way (~140k small-query parses/sec on this machine), so jemalloc is a +pure memory win with no speed cost. + +Reproduce with `bash benchmark/compare-allocators.sh --cycles 3`. A side-by-side against +the WASM backend is available via `node --expose-gc benchmark/memory.mjs --all` once +`@libpg-query/parser` is installed. ## Installation @@ -25,16 +42,45 @@ Native ~4 MB ~70 MB* none* 259k/s npm install @ashbyhq/libpg-query-native ``` -Platform-specific binaries are installed automatically via optional dependencies: -- `@ashbyhq/libpg-query-native-darwin-arm64` -- `@ashbyhq/libpg-query-native-darwin-x64` -- `@ashbyhq/libpg-query-native-linux-x64` -- `@ashbyhq/libpg-query-native-linux-arm64` -- `@ashbyhq/libpg-query-native-linux-x64-musl` -- `@ashbyhq/libpg-query-native-linux-arm64-musl` +Platform-specific binaries ship as separate packages and are installed +automatically via optional dependencies. Each declares `os`/`cpu`/`libc`, so +npm and Yarn install **only** the one matching the host: + +| Package | `os` | `cpu` | `libc` | +|---------|------|-------|--------| +| `@ashbyhq/libpg-query-native-darwin-arm64` | darwin | arm64 | — | +| `@ashbyhq/libpg-query-native-darwin-x64` | darwin | x64 | — | +| `@ashbyhq/libpg-query-native-linux-x64` | linux | x64 | glibc | +| `@ashbyhq/libpg-query-native-linux-arm64` | linux | arm64 | glibc | +| `@ashbyhq/libpg-query-native-linux-x64-musl` | linux | x64 | musl | +| `@ashbyhq/libpg-query-native-linux-arm64-musl` | linux | arm64 | musl | + +glibc and musl builds are marked mutually exclusive, so an Alpine host pulls the +musl binary and a Debian/Ubuntu host pulls the glibc one — never both. No node-gyp or compiler toolchain needed at install time. +### Cross-architecture installs + +When building for a target that differs from the install host (e.g. a Linux +Docker image built on an Apple Silicon Mac), tell the package manager which +architectures to fetch: + +```bash +# npm +npm install --os=linux --cpu=x64 --libc=glibc + +# Yarn Berry — in .yarnrc.yml +supportedArchitectures: + os: [linux] + cpu: [x64] + libc: [glibc] +``` + +> Note: Yarn Classic (1.x) honors `os`/`cpu` but not `libc`. On musl hosts it may +> install both Linux variants; the runtime loader still selects the correct one +> via musl detection. + ## Usage Drop-in replacement for `@libpg-query/parser`: @@ -84,7 +130,7 @@ ENV LD_PRELOAD=/usr/lib/x86_64-linux-gnu/libjemalloc.so.2 # Quick sanity check node --expose-gc benchmark/memory.mjs --small -# Full benchmark (large query, ~2.74 MB SQL) +# Full benchmark (large query, ~3.31 MB SQL) node --expose-gc benchmark/memory.mjs # Compare with/without jemalloc diff --git a/native/benchmark/breakdown.mjs b/native/benchmark/breakdown.mjs new file mode 100644 index 0000000..e2cd44c --- /dev/null +++ b/native/benchmark/breakdown.mjs @@ -0,0 +1,101 @@ +/** + * Measure where time is spent: native parse vs string copy vs JSON.parse. + * + * Usage: + * node benchmark/breakdown.mjs # large query + * node benchmark/breakdown.mjs --small # small query + */ + +import { argv } from "process"; + +const MB = 1024 * 1024; + +function generateBigQuery(unions = 1500, cols = 400) { + const colList = Array.from({ length: cols }, (_, i) => `c${i}`).join(", "); + const select = `SELECT ${colList} FROM t`; + return Array.from({ length: unions }, () => select).join(" UNION ALL "); +} + +async function main() { + const small = argv.includes("--small"); + const addon = (await import("../dist/index.js")).default || await import("../dist/index.js"); + + // Raw addon for string-level measurement + const { createRequire } = await import("module"); + const require = createRequire(import.meta.url); + const raw = require("../prebuilds/" + process.platform + "-" + process.arch + "/libpg_query_native.node"); + + const query = small + ? "SELECT id, name FROM users WHERE id = $1" + : generateBigQuery(); + + console.log(`Query size: ${(query.length / MB).toFixed(2)} MB`); + console.log(""); + + // Warmup + for (let i = 0; i < 5; i++) raw.parseSync(query); + + const iterations = small ? 10000 : 3; + + // 1. Raw native call (parse + JSON serialize in C + string copy to V8) + let jsonStr; + { + const start = performance.now(); + for (let i = 0; i < iterations; i++) { + const res = raw.parseSync(query); + jsonStr = res.result; + } + const elapsed = performance.now() - start; + const avg = elapsed / iterations; + console.log(`Native (parse + JSON serialize + string copy): ${avg < 1 ? (avg * 1000).toFixed(1) + ' μs' : avg.toFixed(2) + ' ms'}`); + console.log(` JSON string size: ${Buffer.byteLength(jsonStr) < MB ? (Buffer.byteLength(jsonStr) / 1024).toFixed(1) + ' KB' : (Buffer.byteLength(jsonStr) / MB).toFixed(2) + ' MB'}`); + } + + // 2. JSON.parse only + { + const start = performance.now(); + for (let i = 0; i < iterations; i++) { + JSON.parse(jsonStr); + } + const elapsed = performance.now() - start; + const avg = elapsed / iterations; + console.log(`JSON.parse: ${avg < 1 ? (avg * 1000).toFixed(1) + ' μs' : avg.toFixed(2) + ' ms'}`); + } + + // 3. End-to-end (native + JSON.parse) + { + const start = performance.now(); + for (let i = 0; i < iterations; i++) { + const res = raw.parseSync(query); + JSON.parse(res.result); + } + const elapsed = performance.now() - start; + const avgE2e = elapsed / iterations; + console.log(`End-to-end (native + JSON.parse): ${avgE2e < 1 ? (avgE2e * 1000).toFixed(1) + ' μs' : avgE2e.toFixed(2) + ' ms'}`); + } + + // 4. Raw native call returning string only (no JSON.parse) — to isolate + let nativeOnly, jsonParseOnly, e2e; + { + const s1 = performance.now(); + for (let i = 0; i < iterations; i++) raw.parseSync(query); + nativeOnly = (performance.now() - s1) / iterations; + + const s2 = performance.now(); + for (let i = 0; i < iterations; i++) JSON.parse(jsonStr); + jsonParseOnly = (performance.now() - s2) / iterations; + + const s3 = performance.now(); + for (let i = 0; i < iterations; i++) { const r = raw.parseSync(query); JSON.parse(r.result); } + e2e = (performance.now() - s3) / iterations; + } + + const fmt = v => v < 1 ? (v * 1000).toFixed(1) + ' μs' : v.toFixed(2) + ' ms'; + console.log(""); + console.log(`Breakdown (second run, warmed up):`); + console.log(` Native (parse+serialize+copy): ${(nativeOnly / e2e * 100).toFixed(0)}% ${fmt(nativeOnly)}`); + console.log(` JSON.parse: ${(jsonParseOnly / e2e * 100).toFixed(0)}% ${fmt(jsonParseOnly)}`); + console.log(` Total e2e: ${fmt(e2e)}`); +} + +main().catch(console.error); diff --git a/native/package.json b/native/package.json index d8247b1..448b5c8 100644 --- a/native/package.json +++ b/native/package.json @@ -12,7 +12,8 @@ } }, "files": [ - "dist/**/*" + "dist/**/*", + "platforms.json" ], "scripts": { "build:native": "make build", diff --git a/native/platforms.json b/native/platforms.json new file mode 100644 index 0000000..bfd3088 --- /dev/null +++ b/native/platforms.json @@ -0,0 +1,38 @@ +{ + "darwin-arm64": { + "os": ["darwin"], + "cpu": ["arm64"], + "runner": "macos-14" + }, + "darwin-x64": { + "os": ["darwin"], + "cpu": ["x64"], + "runner": "macos-13" + }, + "linux-x64": { + "os": ["linux"], + "cpu": ["x64"], + "libc": ["glibc"], + "runner": "ubuntu-24.04" + }, + "linux-arm64": { + "os": ["linux"], + "cpu": ["arm64"], + "libc": ["glibc"], + "runner": "ubuntu-24.04-arm" + }, + "linux-x64-musl": { + "os": ["linux"], + "cpu": ["x64"], + "libc": ["musl"], + "runner": "ubuntu-24.04", + "container": "node:20-alpine" + }, + "linux-arm64-musl": { + "os": ["linux"], + "cpu": ["arm64"], + "libc": ["musl"], + "runner": "ubuntu-24.04-arm", + "container": "node:20-alpine" + } +} diff --git a/native/scripts/package-platforms.mjs b/native/scripts/package-platforms.mjs index 2ef8af2..37a86f8 100644 --- a/native/scripts/package-platforms.mjs +++ b/native/scripts/package-platforms.mjs @@ -6,23 +6,17 @@ * node scripts/package-platforms.mjs */ -import { readdirSync, mkdirSync, copyFileSync, writeFileSync, existsSync } from "fs"; +import { readdirSync, mkdirSync, copyFileSync, writeFileSync, existsSync, readFileSync } from "fs"; import { join, basename } from "path"; -const VERSION = "0.1.0"; +const pkg = JSON.parse(readFileSync("package.json", "utf8")); +const VERSION = pkg.version; const SCOPE = "@ashbyhq"; const BASE_NAME = "libpg-query-native"; const PREBUILDS_DIR = "prebuilds"; const OUT_DIR = "packages"; -const PLATFORM_META = { - "darwin-arm64": { os: ["darwin"], cpu: ["arm64"] }, - "darwin-x64": { os: ["darwin"], cpu: ["x64"] }, - "linux-x64": { os: ["linux"], cpu: ["x64"] }, - "linux-arm64": { os: ["linux"], cpu: ["arm64"] }, - "linux-x64-musl": { os: ["linux"], cpu: ["x64"], libc: ["musl"] }, - "linux-arm64-musl": { os: ["linux"], cpu: ["arm64"], libc: ["musl"] }, -}; +const PLATFORMS = JSON.parse(readFileSync("platforms.json", "utf8")); if (!existsSync(PREBUILDS_DIR)) { console.error(`No ${PREBUILDS_DIR}/ directory found. Run 'make build' first.`); @@ -32,7 +26,7 @@ if (!existsSync(PREBUILDS_DIR)) { const platforms = readdirSync(PREBUILDS_DIR); for (const platform of platforms) { - const meta = PLATFORM_META[platform]; + const meta = PLATFORMS[platform]; if (!meta) { console.warn(`Skipping unknown platform: ${platform}`); continue; diff --git a/native/scripts/sync-optional-deps.mjs b/native/scripts/sync-optional-deps.mjs new file mode 100644 index 0000000..f9b18d6 --- /dev/null +++ b/native/scripts/sync-optional-deps.mjs @@ -0,0 +1,21 @@ +/** + * Sync optionalDependencies in package.json from platforms.json. + * Run after adding/removing platforms: + * node scripts/sync-optional-deps.mjs + */ + +import { readFileSync, writeFileSync } from "fs"; + +const platforms = JSON.parse(readFileSync("platforms.json", "utf8")); +const pkg = JSON.parse(readFileSync("package.json", "utf8")); + +const optionalDeps = {}; +for (const platform of Object.keys(platforms)) { + optionalDeps[`@ashbyhq/libpg-query-native-${platform}`] = pkg.version; +} + +pkg.optionalDependencies = optionalDeps; + +writeFileSync("package.json", JSON.stringify(pkg, null, 2) + "\n"); + +console.log(`Synced ${Object.keys(optionalDeps).length} optionalDependencies from platforms.json`); diff --git a/native/src/index.ts b/native/src/index.ts index bc9901b..43bbf7f 100644 --- a/native/src/index.ts +++ b/native/src/index.ts @@ -62,28 +62,21 @@ function loadNativeAddon(): NativeAddon { // fall through to platform package } - const packageMap: Record = { - "darwin-arm64": "@ashbyhq/libpg-query-native-darwin-arm64", - "darwin-x64": "@ashbyhq/libpg-query-native-darwin-x64", - "linux-x64": "@ashbyhq/libpg-query-native-linux-x64", - "linux-arm64": "@ashbyhq/libpg-query-native-linux-arm64", - "linux-x64-musl": "@ashbyhq/libpg-query-native-linux-x64-musl", - "linux-arm64-musl": "@ashbyhq/libpg-query-native-linux-arm64-musl", - }; - - const pkg = packageMap[platformKey]; - if (!pkg) { + const platforms: Record = require("../platforms.json"); + const pkgName = `@ashbyhq/libpg-query-native-${platformKey}`; + + if (!platforms[platformKey]) { throw new Error( `Unsupported platform: ${platformKey}. ` + - `Supported: ${Object.keys(packageMap).join(", ")}` + `Supported: ${Object.keys(platforms).join(", ")}` ); } try { - return require(pkg); + return require(pkgName); } catch { throw new Error( - `Native addon not found for ${platformKey}. Install ${pkg} or ensure it's in your dependencies.` + `Native addon not found for ${platformKey}. Install ${pkgName} or ensure it's in your dependencies.` ); } } diff --git a/native/test/smoke.mjs b/native/test/smoke.mjs new file mode 100644 index 0000000..05e692e --- /dev/null +++ b/native/test/smoke.mjs @@ -0,0 +1,100 @@ +/** + * Smoke test for the packaged native addon. + * Verifies that the platform package loads and all APIs work. + * + * Usage (from a temp directory with the packages installed): + * node smoke.mjs + * + * Or from the native/ directory with prebuilds in place: + * node test/smoke.mjs + */ + +import { createRequire } from "module"; + +const require = createRequire(import.meta.url); + +let exitCode = 0; + +function assert(condition, message) { + if (!condition) { + console.error(`FAIL: ${message}`); + exitCode = 1; + } +} + +function test(name, fn) { + try { + fn(); + console.log(` ok ${name}`); + } catch (e) { + console.error(` FAIL ${name}: ${e.message}`); + exitCode = 1; + } +} + +const lib = require("@ashbyhq/libpg-query-native"); + +console.log(`Platform: ${process.platform}-${process.arch}`); +console.log(`Node: ${process.version}`); +console.log(""); + +test("parseSync returns parse tree", () => { + const result = lib.parseSync("SELECT 1"); + assert(result.version > 0, "version should be positive"); + assert(result.stmts.length === 1, "should have 1 statement"); + assert(result.stmts[0].stmt.SelectStmt, "should have SelectStmt"); +}); + +test("parse (async) works", async () => { + const result = await lib.parse("SELECT 1"); + assert(result.stmts.length === 1, "should have 1 statement"); +}); + +test("fingerprintSync returns hex string", () => { + const fp = lib.fingerprintSync("SELECT 1"); + assert(typeof fp === "string", "should be string"); + assert(fp.length === 16, "should be 16 chars"); + assert(/^[0-9a-f]+$/.test(fp), "should be hex"); +}); + +test("normalizeSync replaces constants", () => { + const result = lib.normalizeSync("SELECT 1, 'hello'"); + assert(result.includes("$1"), "should have $1"); + assert(!result.includes("hello"), "should not have literal"); +}); + +test("scanSync returns tokens", () => { + const result = lib.scanSync("SELECT id FROM users"); + assert(result.tokens.length > 0, "should have tokens"); + assert(result.tokens[0].text === "SELECT", "first token should be SELECT"); +}); + +test("parsePlPgSQLSync works", () => { + const result = lib.parsePlPgSQLSync( + "CREATE FUNCTION test() RETURNS void AS $$ BEGIN NULL; END; $$ LANGUAGE plpgsql" + ); + assert(result.plpgsql_funcs, "should have plpgsql_funcs"); +}); + +test("parseSync throws SqlError on bad SQL", () => { + try { + lib.parseSync("SELECTT"); + assert(false, "should have thrown"); + } catch (e) { + assert(e.name === "SqlError", `error name should be SqlError, got ${e.name}`); + assert(e.sqlDetails, "should have sqlDetails"); + assert(typeof e.sqlDetails.message === "string", "should have message"); + } +}); + +test("loadModule is a no-op", async () => { + await lib.loadModule(); +}); + +console.log(""); +if (exitCode === 0) { + console.log("All smoke tests passed."); +} else { + console.error("Some smoke tests failed."); +} +process.exit(exitCode); From 62716445d9492670fb1e3494fd8f4fb8a1c6f9c5 Mon Sep 17 00:00:00 2001 From: Ben Asher Date: Sun, 28 Jun 2026 16:47:35 -0700 Subject: [PATCH 04/20] Fix native CI failures Three issues surfaced on the first CI run: - WASM CI (ci.yml) broke because `native` was added to pnpm-workspace.yaml but the committed pnpm-lock.yaml has no native importer, so the frozen-lockfile install failed. The native package is standalone (npm + make, with unpublished platform optionalDependencies) and doesn't belong in the pnpm WASM monorepo. Remove it from the workspace. - glibc Linux jobs ran the Alpine `apk add` step and failed: the matrix derived musl from `!!v.libc`, but glibc platforms now carry `libc: ["glibc"]`. Detect musl via `(v.libc||[]).includes('musl')`. - musl builds wrote to prebuilds/linux-x64 (no suffix) because the Makefile's `ldd --version` parsing didn't detect musl in node:20-alpine, while the JS loader correctly looked in linux-x64-musl. Detect musl via the presence of /lib/ld-musl-*, make PLATFORM_KEY overridable, and pin it from the CI matrix (make build PLATFORM_KEY=). Co-Authored-By: Claude Opus 4.8 --- .github/workflows/native-build.yml | 4 ++-- .github/workflows/native-ci.yml | 2 +- native/Makefile | 9 ++++++--- pnpm-workspace.yaml | 1 - 4 files changed, 9 insertions(+), 7 deletions(-) diff --git a/.github/workflows/native-build.yml b/.github/workflows/native-build.yml index d2fc9bd..1664335 100644 --- a/.github/workflows/native-build.yml +++ b/.github/workflows/native-build.yml @@ -23,7 +23,7 @@ jobs: platform: key, runner: v.runner, container: v.container || '', - musl: !!v.libc, + musl: (v.libc || []).includes('musl'), })); console.log(JSON.stringify(matrix)); ") @@ -57,7 +57,7 @@ jobs: - name: Build native addon working-directory: native - run: make build + run: make build PLATFORM_KEY=${{ matrix.platform }} - name: Build TypeScript working-directory: native diff --git a/.github/workflows/native-ci.yml b/.github/workflows/native-ci.yml index 81a09a1..81e2c42 100644 --- a/.github/workflows/native-ci.yml +++ b/.github/workflows/native-ci.yml @@ -35,7 +35,7 @@ jobs: platform: key, runner: v.runner, container: v.container || '', - musl: !!v.libc, + musl: (v.libc || []).includes('musl'), })); console.log(JSON.stringify(matrix)); ") diff --git a/native/Makefile b/native/Makefile index 15f98e8..57479c9 100644 --- a/native/Makefile +++ b/native/Makefile @@ -22,14 +22,17 @@ else NODE_ARCH := $(ARCH) endif -MUSL := $(shell ldd --version 2>&1 | grep -c musl 2>/dev/null || echo 0) -ifeq ($(MUSL),1) +# Detect musl libc (Alpine) by the presence of its dynamic loader. +# More reliable than parsing `ldd --version`, which varies across distros. +ifneq ($(wildcard /lib/ld-musl-*),) PLATFORM_SUFFIX := -musl else PLATFORM_SUFFIX := endif -PLATFORM_KEY := $(PLATFORM)-$(NODE_ARCH)$(PLATFORM_SUFFIX) +# PLATFORM_KEY can be overridden (e.g. `make build PLATFORM_KEY=linux-x64-musl`) +# so CI can pin the output dir to the known target instead of relying on detection. +PLATFORM_KEY ?= $(PLATFORM)-$(NODE_ARCH)$(PLATFORM_SUFFIX) CACHE_DIR := .cache/$(PLATFORM_KEY) LIBPG_QUERY_DIR := $(CACHE_DIR)/libpg_query diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml index 2eae5cc..7135354 100644 --- a/pnpm-workspace.yaml +++ b/pnpm-workspace.yaml @@ -1,7 +1,6 @@ packages: - 'parser' - 'full' - - 'native' - 'versions/18' - 'versions/17' - 'versions/16' From 77bd0a5929bb40d291da3a5ba4de70f104196dad Mon Sep 17 00:00:00 2001 From: Ben Asher Date: Sun, 28 Jun 2026 17:15:05 -0700 Subject: [PATCH 05/20] Harden native package-test job The package-test job only runs after builds pass, so it wasn't exercised on earlier runs. Reworked it to be robust and validated the full flow locally: - Install via `npm pack` tarballs (main + platform) instead of installing the local source dir. Installing the source pulls the unpublished platform optionalDependencies from the registry (404s); tarballs + `--omit=optional` avoid that and install the matching platform binary explicitly. - `npm pack --pack-destination` does not create its target dir, so `mkdir -p` it first (this was the concrete failure reproduced locally). - Drop `--install-strategy=nested`; default hoisting resolves correctly. Co-Authored-By: Claude Opus 4.8 --- .github/workflows/native-ci.yml | 21 +++++++++++++++------ native/.gitignore | 1 + 2 files changed, 16 insertions(+), 6 deletions(-) diff --git a/.github/workflows/native-ci.yml b/.github/workflows/native-ci.yml index 81e2c42..3064745 100644 --- a/.github/workflows/native-ci.yml +++ b/.github/workflows/native-ci.yml @@ -69,22 +69,31 @@ jobs: name: prebuild-${{ matrix.platform }} path: native/prebuilds/${{ matrix.platform }} - - name: Install, build TS, and generate platform package + - name: Build TS and pack tarballs working-directory: native run: | npm install npx tsc node scripts/package-platforms.mjs + # Pack the main package and this platform's package into tarballs. + # Tarballs install cleanly without registry lookups for the (unpublished) + # platform optionalDependencies. --pack-destination requires the dir to exist. + mkdir -p "$GITHUB_WORKSPACE/native/dist-packs" + npm pack --pack-destination "$GITHUB_WORKSPACE/native/dist-packs" + (cd "packages/libpg-query-native-${{ matrix.platform }}" \ + && npm pack --pack-destination "$GITHUB_WORKSPACE/native/dist-packs") + echo "Packed:"; ls -1 "$GITHUB_WORKSPACE/native/dist-packs" - name: Install package in isolated directory run: | mkdir -p /tmp/smoke-test cd /tmp/smoke-test - npm init -y - npm install "$GITHUB_WORKSPACE/native" --install-strategy=nested - # Install the platform package from generated dir - platform_pkg=$(ls -d "$GITHUB_WORKSPACE/native/packages/"*${{ matrix.platform }}*) - npm install "$platform_pkg" --install-strategy=nested + npm init -y >/dev/null + # --omit=optional skips the main package's unpublished platform deps; + # the matching platform tarball is installed explicitly alongside it. + npm install --omit=optional "$GITHUB_WORKSPACE/native/dist-packs/"*.tgz + echo "Installed @ashbyhq packages:" + ls node_modules/@ashbyhq - name: Run smoke test run: | diff --git a/native/.gitignore b/native/.gitignore index 1176e91..55014c9 100644 --- a/native/.gitignore +++ b/native/.gitignore @@ -4,3 +4,4 @@ packages/ dist/ node_modules/ package-lock.json +dist-packs/ From a1e51923c8da76ddf4099b1b8906ae65bc9f0464 Mon Sep 17 00:00:00 2001 From: Ben Asher Date: Sun, 28 Jun 2026 17:19:56 -0700 Subject: [PATCH 06/20] Bump GitHub Actions to latest major versions MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - actions/checkout v4 -> v5 (Node 24 runtime) - actions/setup-node v4 -> v6 (latest, v6.2.0) - actions/upload-artifact v4 -> v5 - actions/download-artifact v4 -> v5 (kept in lockstep with upload; the artifact actions require matching majors for up/download) setup-node v5+ auto-enables dependency caching, which errors when no lockfile is present — and native/ doesn't commit package-lock.json. Set package-manager-cache: false on each setup-node to preserve v4 behavior. Co-Authored-By: Claude Opus 4.8 --- .github/workflows/native-build.yml | 9 +++++---- .github/workflows/native-ci.yml | 9 +++++---- .github/workflows/native-release.yml | 7 ++++--- 3 files changed, 14 insertions(+), 11 deletions(-) diff --git a/.github/workflows/native-build.yml b/.github/workflows/native-build.yml index 1664335..fb1f2ae 100644 --- a/.github/workflows/native-build.yml +++ b/.github/workflows/native-build.yml @@ -14,7 +14,7 @@ jobs: outputs: include: ${{ steps.gen.outputs.include }} steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v5 - id: gen run: | include=$(node -e " @@ -43,13 +43,14 @@ jobs: if: matrix.musl run: apk add --no-cache git make g++ python3 - - uses: actions/checkout@v4 + - uses: actions/checkout@v5 - name: Setup Node.js if: "!matrix.musl" - uses: actions/setup-node@v4 + uses: actions/setup-node@v6 with: node-version: "20" + package-manager-cache: false - name: Install dependencies working-directory: native @@ -68,7 +69,7 @@ jobs: run: npm test - name: Upload prebuild - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v5 with: name: prebuild-${{ matrix.platform }} path: native/prebuilds/${{ matrix.platform }}/libpg_query_native.node diff --git a/.github/workflows/native-ci.yml b/.github/workflows/native-ci.yml index 3064745..1de130c 100644 --- a/.github/workflows/native-ci.yml +++ b/.github/workflows/native-ci.yml @@ -26,7 +26,7 @@ jobs: outputs: include: ${{ steps.gen.outputs.include }} steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v5 - id: gen run: | include=$(node -e " @@ -55,16 +55,17 @@ jobs: if: matrix.musl run: apk add --no-cache git - - uses: actions/checkout@v4 + - uses: actions/checkout@v5 - name: Setup Node.js if: "!matrix.musl" - uses: actions/setup-node@v4 + uses: actions/setup-node@v6 with: node-version: "20" + package-manager-cache: false - name: Download prebuild - uses: actions/download-artifact@v4 + uses: actions/download-artifact@v5 with: name: prebuild-${{ matrix.platform }} path: native/prebuilds/${{ matrix.platform }} diff --git a/.github/workflows/native-release.yml b/.github/workflows/native-release.yml index 5944088..d4b0511 100644 --- a/.github/workflows/native-release.yml +++ b/.github/workflows/native-release.yml @@ -25,15 +25,16 @@ jobs: permissions: contents: write steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v5 - - uses: actions/setup-node@v4 + - uses: actions/setup-node@v6 with: node-version: "20" + package-manager-cache: false registry-url: "https://registry.npmjs.org" - name: Download all prebuilds - uses: actions/download-artifact@v4 + uses: actions/download-artifact@v5 with: path: native/prebuilds pattern: prebuild-* From 16ec2e987ecc6cf7c31110e8b56b39b075b80005 Mon Sep 17 00:00:00 2001 From: Ben Asher Date: Sun, 28 Jun 2026 17:22:54 -0700 Subject: [PATCH 07/20] README: real native-vs-WASM comparison with measured numbers Replace the native-only allocator table with a 3-way comparison (WASM, native system malloc, native jemalloc) using isolated per-process measurements: WASM peaks at 1359 MB and never releases (+1202 MB retained), native+jemalloc peaks at 498 MB and stabilizes (+377 MB), and native is ~11% faster on throughput. Co-Authored-By: Claude Opus 4.8 --- native/README.md | 44 ++++++++++++++++++++++++++------------------ 1 file changed, 26 insertions(+), 18 deletions(-) diff --git a/native/README.md b/native/README.md index 54d1c34..847103b 100644 --- a/native/README.md +++ b/native/README.md @@ -15,26 +15,34 @@ actually be returned to the OS. Pairing it with **jemalloc** (via `LD_PRELOAD` / `DYLD_INSERT_LIBRARIES`) roughly halves peak RSS and, more importantly, keeps it *stable* across repeated large parses instead of ratcheting. -### Measured: native, system malloc vs jemalloc +### Measured: native vs WASM Parsing a 3.31 MB SQL query (1500× `UNION ALL`, ~65 MB JSON parse tree), 3 parse/free -cycles, `darwin-arm64`, Node 24. Peak is the max RSS during a parse; "after" is RSS -once the result is dropped and GC settles. - -| Allocator | cycle 1 peak | cycle 2 peak | cycle 3 peak | retained after | -|-----------|-------------|-------------|-------------|----------------| -| system malloc | 649 MB | 867 MB | **932 MB** | 867 MB | -| jemalloc | 381 MB | 497 MB | **498 MB** | 433 MB | -| jemalloc (`dirty_decay_ms:0,muzzy_decay_ms:0`) | 377 MB | 477 MB | **477 MB** | 411 MB | - -System malloc climbs every cycle (649 → 932 MB) — it fragments and holds the freed -arenas. jemalloc stabilizes at ~498 MB by cycle 2 and stays there. Throughput is -identical either way (~140k small-query parses/sec on this machine), so jemalloc is a -pure memory win with no speed cost. - -Reproduce with `bash benchmark/compare-allocators.sh --cycles 3`. A side-by-side against -the WASM backend is available via `node --expose-gc benchmark/memory.mjs --all` once -`@libpg-query/parser` is installed. +cycles, `darwin-arm64`, Node 24. Each backend measured in its own process. "Retained" +is RSS after the result is dropped and GC settles; throughput is a small query ×10k. + +| Backend | idle RSS | peak RSS (max of 3) | retained after free | throughput | +|---------|---------|---------------------|---------------------|------------| +| WASM (`@libpg-query/parser`) | 93 MB | 1359 MB | **+1202 MB** (never shrinks) | 125k/s | +| Native — system malloc | 53 MB | 932 MB | +812 MB (ratchets up) | 139k/s | +| Native — **jemalloc** | 55 MB | **498 MB** | **+377 MB** (stabilizes) | 139k/s | + +Per-cycle peak progression: + +``` +WASM: 1261 → 1359 → 1359 MB (plateaus at a high permanent floor) +Native system: 649 → 867 → 932 MB (fragments, still climbing) +Native jemalloc: 381 → 497 → 498 MB (flat after cycle 2) +``` + +WASM linear memory only ever grows, so ~1.2 GB from one big parse is held for the +process lifetime. Native + system malloc is lower but still ratchets. Native + +jemalloc has ~2.7× lower peak than WASM, returns freed pages to the OS, and stabilizes. +`MALLOC_CONF=dirty_decay_ms:0,muzzy_decay_ms:0` trims peak a little further (~477 MB). +Throughput is identical across allocators — jemalloc is a pure memory win. + +Reproduce with `node --expose-gc benchmark/memory.mjs --all --cycles 3` (with +`@libpg-query/parser` installed) and `bash benchmark/compare-allocators.sh --cycles 3`. ## Installation From 9412972bba5de3d2e59dda82b6e1402d0c081e95 Mon Sep 17 00:00:00 2001 From: Ben Asher Date: Sun, 28 Jun 2026 20:56:21 -0700 Subject: [PATCH 08/20] Update GitHub Actions to current latest majors Verified against each action's releases/latest: - actions/checkout v5 -> v7 - actions/upload-artifact v5 -> v7 - actions/download-artifact v5 -> v8 (upload v7 + download v8 is the current compatible pair per the GitHub Actions changelog; default archive:true keeps existing zip behavior, so no flow change) - actions/setup-node stays v6 (latest is v6.4.0) Co-Authored-By: Claude Opus 4.8 --- .github/workflows/native-build.yml | 6 +++--- .github/workflows/native-ci.yml | 6 +++--- .github/workflows/native-release.yml | 4 ++-- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/workflows/native-build.yml b/.github/workflows/native-build.yml index fb1f2ae..ff78c5d 100644 --- a/.github/workflows/native-build.yml +++ b/.github/workflows/native-build.yml @@ -14,7 +14,7 @@ jobs: outputs: include: ${{ steps.gen.outputs.include }} steps: - - uses: actions/checkout@v5 + - uses: actions/checkout@v7 - id: gen run: | include=$(node -e " @@ -43,7 +43,7 @@ jobs: if: matrix.musl run: apk add --no-cache git make g++ python3 - - uses: actions/checkout@v5 + - uses: actions/checkout@v7 - name: Setup Node.js if: "!matrix.musl" @@ -69,7 +69,7 @@ jobs: run: npm test - name: Upload prebuild - uses: actions/upload-artifact@v5 + uses: actions/upload-artifact@v7 with: name: prebuild-${{ matrix.platform }} path: native/prebuilds/${{ matrix.platform }}/libpg_query_native.node diff --git a/.github/workflows/native-ci.yml b/.github/workflows/native-ci.yml index 1de130c..38fe42f 100644 --- a/.github/workflows/native-ci.yml +++ b/.github/workflows/native-ci.yml @@ -26,7 +26,7 @@ jobs: outputs: include: ${{ steps.gen.outputs.include }} steps: - - uses: actions/checkout@v5 + - uses: actions/checkout@v7 - id: gen run: | include=$(node -e " @@ -55,7 +55,7 @@ jobs: if: matrix.musl run: apk add --no-cache git - - uses: actions/checkout@v5 + - uses: actions/checkout@v7 - name: Setup Node.js if: "!matrix.musl" @@ -65,7 +65,7 @@ jobs: package-manager-cache: false - name: Download prebuild - uses: actions/download-artifact@v5 + uses: actions/download-artifact@v8 with: name: prebuild-${{ matrix.platform }} path: native/prebuilds/${{ matrix.platform }} diff --git a/.github/workflows/native-release.yml b/.github/workflows/native-release.yml index d4b0511..b9c9a2d 100644 --- a/.github/workflows/native-release.yml +++ b/.github/workflows/native-release.yml @@ -25,7 +25,7 @@ jobs: permissions: contents: write steps: - - uses: actions/checkout@v5 + - uses: actions/checkout@v7 - uses: actions/setup-node@v6 with: @@ -34,7 +34,7 @@ jobs: registry-url: "https://registry.npmjs.org" - name: Download all prebuilds - uses: actions/download-artifact@v5 + uses: actions/download-artifact@v8 with: path: native/prebuilds pattern: prebuild-* From 58d3f531ed1b62ba45df1b03e9cb026c16102d4d Mon Sep 17 00:00:00 2001 From: Ben Asher Date: Sun, 28 Jun 2026 21:01:17 -0700 Subject: [PATCH 09/20] Fix musl CI: run Alpine builds via docker, not job container MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit JavaScript actions (checkout/setup-node/upload-artifact) can't run inside an Alpine container on arm64 runners — GitHub injects a glibc Node that fails with "JavaScript Actions in Alpine containers are only supported on x64 Linux runners." This broke the linux-arm64-musl build. Drop the job-level `container: node:20-alpine` for both the build and package-test jobs. The host actions now run on the glibc runner, and the musl build/test/smoke runs via `docker run "${{ matrix.container }}"` — which pulls the arch-matching Alpine image on both x64 and arm64 runners. Validated the full arm64-musl flow locally (Apple Silicon → arm64 Alpine): libpg_query + addon compile, 46 tests pass, output is a musl aarch64 ELF. Co-Authored-By: Claude Opus 4.8 --- .github/workflows/native-build.yml | 41 ++++++++++---------- .github/workflows/native-ci.yml | 61 ++++++++++++++++-------------- 2 files changed, 54 insertions(+), 48 deletions(-) diff --git a/.github/workflows/native-build.yml b/.github/workflows/native-build.yml index ff78c5d..7143f08 100644 --- a/.github/workflows/native-build.yml +++ b/.github/workflows/native-build.yml @@ -37,36 +37,39 @@ jobs: fail-fast: false matrix: include: ${{ fromJSON(needs.matrix.outputs.include) }} - container: ${{ matrix.container || '' }} + # No job-level `container`: JavaScript actions (checkout/setup-node/upload) + # can't run inside an Alpine container on arm64 runners. musl builds run via + # `docker run` instead, with the host actions executing on the glibc host. steps: - - name: Install Alpine build deps - if: matrix.musl - run: apk add --no-cache git make g++ python3 - - uses: actions/checkout@v7 - name: Setup Node.js - if: "!matrix.musl" + if: ${{ !matrix.musl }} uses: actions/setup-node@v6 with: node-version: "20" package-manager-cache: false - - name: Install dependencies - working-directory: native - run: npm install - - - name: Build native addon - working-directory: native - run: make build PLATFORM_KEY=${{ matrix.platform }} - - - name: Build TypeScript + - name: Build and test (glibc / macOS) + if: ${{ !matrix.musl }} working-directory: native - run: npx tsc + run: | + npm install + make build PLATFORM_KEY=${{ matrix.platform }} + npx tsc + npm test - - name: Run tests - working-directory: native - run: npm test + - name: Build and test (musl, in Alpine container) + if: ${{ matrix.musl }} + run: | + docker run --rm -v "$GITHUB_WORKSPACE/native:/work" -w /work \ + "${{ matrix.container }}" sh -c ' + apk add --no-cache git make g++ python3 && + npm install && + make build PLATFORM_KEY=${{ matrix.platform }} && + npx tsc && + npm test + ' - name: Upload prebuild uses: actions/upload-artifact@v7 diff --git a/.github/workflows/native-ci.yml b/.github/workflows/native-ci.yml index 38fe42f..afa8e43 100644 --- a/.github/workflows/native-ci.yml +++ b/.github/workflows/native-ci.yml @@ -49,16 +49,14 @@ jobs: fail-fast: false matrix: include: ${{ fromJSON(needs.package-test-matrix.outputs.include) }} - container: ${{ matrix.container || '' }} + # No job-level `container` (JS actions can't run in an Alpine container on + # arm64). The prebuild is downloaded on the glibc host; the musl smoke test + # runs in the Alpine image via `docker run`. steps: - - name: Install Alpine deps - if: matrix.musl - run: apk add --no-cache git - - uses: actions/checkout@v7 - name: Setup Node.js - if: "!matrix.musl" + if: ${{ !matrix.musl }} uses: actions/setup-node@v6 with: node-version: "20" @@ -70,34 +68,39 @@ jobs: name: prebuild-${{ matrix.platform }} path: native/prebuilds/${{ matrix.platform }} - - name: Build TS and pack tarballs - working-directory: native + # Pack the main + platform packages into tarballs and install them in an + # isolated dir. Tarballs + --omit=optional install cleanly without registry + # lookups for the (unpublished) platform optionalDependencies. + - name: Pack, install, smoke test (glibc / macOS) + if: ${{ !matrix.musl }} run: | + cd native npm install npx tsc node scripts/package-platforms.mjs - # Pack the main package and this platform's package into tarballs. - # Tarballs install cleanly without registry lookups for the (unpublished) - # platform optionalDependencies. --pack-destination requires the dir to exist. - mkdir -p "$GITHUB_WORKSPACE/native/dist-packs" - npm pack --pack-destination "$GITHUB_WORKSPACE/native/dist-packs" + mkdir -p dist-packs + npm pack --pack-destination "$PWD/dist-packs" (cd "packages/libpg-query-native-${{ matrix.platform }}" \ - && npm pack --pack-destination "$GITHUB_WORKSPACE/native/dist-packs") - echo "Packed:"; ls -1 "$GITHUB_WORKSPACE/native/dist-packs" - - - name: Install package in isolated directory - run: | - mkdir -p /tmp/smoke-test - cd /tmp/smoke-test - npm init -y >/dev/null - # --omit=optional skips the main package's unpublished platform deps; - # the matching platform tarball is installed explicitly alongside it. + && npm pack --pack-destination "$PWD/../../dist-packs") + mkdir -p /tmp/smoke-test && cd /tmp/smoke-test && npm init -y >/dev/null npm install --omit=optional "$GITHUB_WORKSPACE/native/dist-packs/"*.tgz - echo "Installed @ashbyhq packages:" - ls node_modules/@ashbyhq - - - name: Run smoke test - run: | - cd /tmp/smoke-test + echo "Installed @ashbyhq packages:"; ls node_modules/@ashbyhq cp "$GITHUB_WORKSPACE/native/test/smoke.mjs" smoke.mjs node smoke.mjs + + - name: Pack, install, smoke test (musl, in Alpine container) + if: ${{ matrix.musl }} + run: | + docker run --rm -v "$GITHUB_WORKSPACE/native:/work" -w /work \ + "${{ matrix.container }}" sh -c ' + apk add --no-cache git && + npm install && npx tsc && node scripts/package-platforms.mjs && + mkdir -p dist-packs && + npm pack --pack-destination /work/dist-packs && + (cd packages/libpg-query-native-${{ matrix.platform }} && npm pack --pack-destination /work/dist-packs) && + mkdir -p /tmp/smoke && cd /tmp/smoke && npm init -y >/dev/null && + npm install --omit=optional /work/dist-packs/*.tgz && + echo "Installed @ashbyhq packages:" && ls node_modules/@ashbyhq && + cp /work/test/smoke.mjs smoke.mjs && + node smoke.mjs + ' From 15ffd0f07469fa0f775f0e9c03e64253d59653d2 Mon Sep 17 00:00:00 2001 From: Ben Asher Date: Sun, 28 Jun 2026 21:18:06 -0700 Subject: [PATCH 10/20] Use Node 24, drop darwin-x64, minimize macOS usage - Bump all builds to Node 24: setup-node node-version 24, and the musl Alpine images to node:24-alpine (via platforms.json container). - Drop the darwin-x64 target. Apple Silicon (darwin-arm64) is the only macOS variant built; the sole remaining macOS runner (macos-14) is used only for that explicit variant. Everything else runs on Ubuntu. - Re-sync optionalDependencies and update the platform table accordingly. Targets are now: darwin-arm64, linux-x64, linux-arm64, linux-x64-musl, linux-arm64-musl. Co-Authored-By: Claude Opus 4.8 --- .github/workflows/native-build.yml | 2 +- .github/workflows/native-ci.yml | 2 +- .github/workflows/native-release.yml | 2 +- native/README.md | 1 - native/package.json | 1 - native/platforms.json | 9 ++------- 6 files changed, 5 insertions(+), 12 deletions(-) diff --git a/.github/workflows/native-build.yml b/.github/workflows/native-build.yml index 7143f08..e42aa67 100644 --- a/.github/workflows/native-build.yml +++ b/.github/workflows/native-build.yml @@ -47,7 +47,7 @@ jobs: if: ${{ !matrix.musl }} uses: actions/setup-node@v6 with: - node-version: "20" + node-version: "24" package-manager-cache: false - name: Build and test (glibc / macOS) diff --git a/.github/workflows/native-ci.yml b/.github/workflows/native-ci.yml index afa8e43..f86ce06 100644 --- a/.github/workflows/native-ci.yml +++ b/.github/workflows/native-ci.yml @@ -59,7 +59,7 @@ jobs: if: ${{ !matrix.musl }} uses: actions/setup-node@v6 with: - node-version: "20" + node-version: "24" package-manager-cache: false - name: Download prebuild diff --git a/.github/workflows/native-release.yml b/.github/workflows/native-release.yml index b9c9a2d..7e0411b 100644 --- a/.github/workflows/native-release.yml +++ b/.github/workflows/native-release.yml @@ -29,7 +29,7 @@ jobs: - uses: actions/setup-node@v6 with: - node-version: "20" + node-version: "24" package-manager-cache: false registry-url: "https://registry.npmjs.org" diff --git a/native/README.md b/native/README.md index 847103b..4427aa3 100644 --- a/native/README.md +++ b/native/README.md @@ -57,7 +57,6 @@ npm and Yarn install **only** the one matching the host: | Package | `os` | `cpu` | `libc` | |---------|------|-------|--------| | `@ashbyhq/libpg-query-native-darwin-arm64` | darwin | arm64 | — | -| `@ashbyhq/libpg-query-native-darwin-x64` | darwin | x64 | — | | `@ashbyhq/libpg-query-native-linux-x64` | linux | x64 | glibc | | `@ashbyhq/libpg-query-native-linux-arm64` | linux | arm64 | glibc | | `@ashbyhq/libpg-query-native-linux-x64-musl` | linux | x64 | musl | diff --git a/native/package.json b/native/package.json index 448b5c8..299dd92 100644 --- a/native/package.json +++ b/native/package.json @@ -49,7 +49,6 @@ }, "optionalDependencies": { "@ashbyhq/libpg-query-native-darwin-arm64": "0.1.0", - "@ashbyhq/libpg-query-native-darwin-x64": "0.1.0", "@ashbyhq/libpg-query-native-linux-x64": "0.1.0", "@ashbyhq/libpg-query-native-linux-arm64": "0.1.0", "@ashbyhq/libpg-query-native-linux-x64-musl": "0.1.0", diff --git a/native/platforms.json b/native/platforms.json index bfd3088..994b659 100644 --- a/native/platforms.json +++ b/native/platforms.json @@ -4,11 +4,6 @@ "cpu": ["arm64"], "runner": "macos-14" }, - "darwin-x64": { - "os": ["darwin"], - "cpu": ["x64"], - "runner": "macos-13" - }, "linux-x64": { "os": ["linux"], "cpu": ["x64"], @@ -26,13 +21,13 @@ "cpu": ["x64"], "libc": ["musl"], "runner": "ubuntu-24.04", - "container": "node:20-alpine" + "container": "node:24-alpine" }, "linux-arm64-musl": { "os": ["linux"], "cpu": ["arm64"], "libc": ["musl"], "runner": "ubuntu-24.04-arm", - "container": "node:20-alpine" + "container": "node:24-alpine" } } From 05f1fce33212168ad6fd9405d543177366f44887 Mon Sep 17 00:00:00 2001 From: Ben Asher Date: Sun, 28 Jun 2026 21:33:41 -0700 Subject: [PATCH 11/20] Drop WASM CI build workflows (keep WASM source) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Remove the WASM build/test workflows — ci.yml, build-wasm.yml, and build-wasm-no-docker.yaml — so no WASM builds run in CI. The WASM packages (full/, parser/, versions/, types/, enums/, protos/) and the rest of the monorepo are kept intact; they can still be built manually. Only the native backend's workflows (native-ci.yml, native-build.yml, native-release.yml) run in CI now. This reverts the full WASM removal from the previous commit, keeping WASM support while dropping just its automated builds. Co-Authored-By: Claude Opus 4.8 --- .github/workflows/build-wasm-no-docker.yaml | 56 ---- .github/workflows/build-wasm.yml | 48 --- .github/workflows/ci.yml | 310 -------------------- 3 files changed, 414 deletions(-) delete mode 100644 .github/workflows/build-wasm-no-docker.yaml delete mode 100644 .github/workflows/build-wasm.yml delete mode 100644 .github/workflows/ci.yml diff --git a/.github/workflows/build-wasm-no-docker.yaml b/.github/workflows/build-wasm-no-docker.yaml deleted file mode 100644 index 71d2029..0000000 --- a/.github/workflows/build-wasm-no-docker.yaml +++ /dev/null @@ -1,56 +0,0 @@ -name: Build Wasm No Docker 🛠 -'on': - workflow_dispatch: null -jobs: - build-wasm-no-docker: - runs-on: ubuntu-latest - steps: - - name: Checkout Repository 📥 - uses: actions/checkout@v4 - - name: Setup Node.js 🌐 - uses: actions/setup-node@v4 - with: - node-version: 20.x - - - name: Setup pnpm 📦 - uses: pnpm/action-setup@v2 - with: - version: 8.15.0 - - - name: Get pnpm store directory 📁 - shell: bash - run: | - echo "STORE_PATH=$(pnpm store path --silent)" >> $GITHUB_ENV - - - name: Setup pnpm cache 🗄️ - uses: actions/cache@v3 - with: - path: ${{ env.STORE_PATH }} - key: ${{ runner.os }}-pnpm-store-${{ hashFiles('**/pnpm-lock.yaml') }} - restore-keys: | - ${{ runner.os }}-pnpm-store- - - - name: Install Dependencies 🧶 - run: pnpm install - - - name: Install Emscripten ✍🏻 - run: | - sudo apt-get update - sudo apt-get install cmake python3 python3-pip - git clone --branch 3.1.59 --depth 1 https://github.com/emscripten-core/emsdk.git - cd emsdk - ./emsdk install 3.1.59 - ./emsdk activate 3.1.59 - source ./emsdk_env.sh - working-directory: full - - name: Build with Emscripten 🏗 - run: | - source ./emsdk/emsdk_env.sh - emmake make - emmake make build - working-directory: full - - name: Archive production artifacts 🏛 - uses: actions/upload-artifact@v4 - with: - name: wasm-artifacts - path: full/wasm diff --git a/.github/workflows/build-wasm.yml b/.github/workflows/build-wasm.yml deleted file mode 100644 index d520032..0000000 --- a/.github/workflows/build-wasm.yml +++ /dev/null @@ -1,48 +0,0 @@ -name: Build Wasm 🛠 - -on: - workflow_dispatch: - -jobs: - build-wasm: - runs-on: macos-latest - steps: - - name: Checkout Repository 📥 - uses: actions/checkout@v4 - - - name: Setup Node.js 🌐 - uses: actions/setup-node@v4 - with: - node-version: '20.x' - - - name: Setup pnpm 📦 - uses: pnpm/action-setup@v2 - with: - version: 8.15.0 - - - name: Get pnpm store directory 📁 - shell: bash - run: | - echo "STORE_PATH=$(pnpm store path --silent)" >> $GITHUB_ENV - - - name: Setup pnpm cache 🗄️ - uses: actions/cache@v3 - with: - path: ${{ env.STORE_PATH }} - key: ${{ runner.os }}-pnpm-store-${{ hashFiles('**/pnpm-lock.yaml') }} - restore-keys: | - ${{ runner.os }}-pnpm-store- - - - name: Install Dependencies 🧶 - run: pnpm install - - - name: Build WASM 🏗 - run: pnpm run build - working-directory: full - - - name: Archive production artifacts 🏛 - uses: actions/upload-artifact@v4 - with: - name: wasm-artifacts - path: full/wasm/ - retention-days: 7 diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml deleted file mode 100644 index dac525b..0000000 --- a/.github/workflows/ci.yml +++ /dev/null @@ -1,310 +0,0 @@ -name: CI 🚀 - -on: - pull_request: - types: [opened, synchronize, reopened] - push: - workflow_dispatch: - -jobs: - build-wasm: - name: Build WASM ${{ matrix.package }} 🔧 - runs-on: ubuntu-latest - strategy: - matrix: - package: - - { name: 'full', path: 'full', version: '18' } - - { name: 'v13', path: 'versions/13', version: '13' } - - { name: 'v14', path: 'versions/14', version: '14' } - - { name: 'v15', path: 'versions/15', version: '15' } - - { name: 'v16', path: 'versions/16', version: '16' } - - { name: 'v17', path: 'versions/17', version: '17' } - - { name: 'v18', path: 'versions/18', version: '18' } - fail-fast: false - steps: - - name: Checkout Repository 📥 - uses: actions/checkout@v4 - - - name: Setup Node.js 🌐 - uses: actions/setup-node@v4 - with: - node-version: '20.x' - - - name: Setup pnpm 📦 - uses: pnpm/action-setup@v2 - with: - version: 8.15.1 - - - name: Get pnpm store directory 📁 - shell: bash - run: | - echo "STORE_PATH=$(pnpm store path --silent)" >> $GITHUB_ENV - - - name: Setup pnpm cache 🗄️ - uses: actions/cache@v3 - with: - path: ${{ env.STORE_PATH }} - key: ${{ runner.os }}-pnpm-store-${{ hashFiles('**/pnpm-lock.yaml') }} - restore-keys: | - ${{ runner.os }}-pnpm-store- - - - name: Install Dependencies 🧶 - run: pnpm install - - - name: Build WASM 🏗 - run: | - if [ "${{ matrix.package.name }}" = "v13" ]; then - # Download prebuilt WASM for v13 since it can't build in CI - mkdir -p wasm - curl -o v13.tgz "https://registry.npmjs.org/@libpg-query/v13/-/v13-13.5.7.tgz" - tar -xzf v13.tgz --strip-components=1 package/wasm - rm v13.tgz - else - pnpm run build - fi - working-directory: ${{ matrix.package.path }} - - - name: Upload WASM Artifacts 📦 - uses: actions/upload-artifact@v4 - with: - name: wasm-artifacts-${{ matrix.package.name }} - path: ${{ matrix.package.path }}/wasm/ - retention-days: 1 - - test: - name: Test ${{ matrix.package.name }} on ${{ matrix.os }} ${{ matrix.os == 'ubuntu-latest' && '🐧' || matrix.os == 'macos-latest' && '🍎' || '🪟' }} - needs: build-wasm - strategy: - matrix: - os: [ubuntu-latest, macos-latest, windows-latest] - package: - - { name: 'full', path: 'full', version: '18' } - - { name: 'v13', path: 'versions/13', version: '13' } - - { name: 'v14', path: 'versions/14', version: '14' } - - { name: 'v15', path: 'versions/15', version: '15' } - - { name: 'v16', path: 'versions/16', version: '16' } - - { name: 'v17', path: 'versions/17', version: '17' } - - { name: 'v18', path: 'versions/18', version: '18' } - fail-fast: false - runs-on: ${{ matrix.os }} - steps: - - name: Checkout Repository 📥 - uses: actions/checkout@v4 - - - name: Setup Node.js 🌐 - uses: actions/setup-node@v4 - with: - node-version: '20.x' - - - name: Setup pnpm 📦 - uses: pnpm/action-setup@v2 - with: - version: 8.15.1 - - - name: Get pnpm store directory 📁 - shell: bash - run: | - echo "STORE_PATH=$(pnpm store path --silent)" >> $GITHUB_ENV - - - name: Setup pnpm cache 🗄️ - uses: actions/cache@v3 - with: - path: ${{ env.STORE_PATH }} - key: ${{ runner.os }}-pnpm-store-${{ hashFiles('**/pnpm-lock.yaml') }} - restore-keys: | - ${{ runner.os }}-pnpm-store- - - - name: Install Dependencies 🧶 - run: pnpm install - - - name: Download WASM Artifacts 📥 - uses: actions/download-artifact@v4 - with: - name: wasm-artifacts-${{ matrix.package.name }} - path: ${{ matrix.package.path }}/wasm/ - - - name: Run Tests 🔍 - run: pnpm run test - working-directory: ${{ matrix.package.path }} - - build-parser: - name: Build Parser Package 📦 - needs: build-wasm - runs-on: ubuntu-latest - steps: - - name: Checkout Repository 📥 - uses: actions/checkout@v4 - - - name: Setup Node.js 🌐 - uses: actions/setup-node@v4 - with: - node-version: '20.x' - - - name: Setup pnpm 📦 - uses: pnpm/action-setup@v2 - with: - version: 8.15.1 - - - name: Get pnpm store directory 📁 - shell: bash - run: | - echo "STORE_PATH=$(pnpm store path --silent)" >> $GITHUB_ENV - - - name: Setup pnpm cache 🗄️ - uses: actions/cache@v3 - with: - path: ${{ env.STORE_PATH }} - key: ${{ runner.os }}-pnpm-store-${{ hashFiles('**/pnpm-lock.yaml') }} - restore-keys: | - ${{ runner.os }}-pnpm-store- - - - name: Install Dependencies 🧶 - run: pnpm install - - - name: Install Parser Dependencies 📦 - run: pnpm install - working-directory: parser - - - name: Download v13 WASM Artifacts 📥 - uses: actions/download-artifact@v4 - with: - name: wasm-artifacts-v13 - path: versions/13/wasm/ - - - name: Download v14 WASM Artifacts 📥 - uses: actions/download-artifact@v4 - with: - name: wasm-artifacts-v14 - path: versions/14/wasm/ - - - name: Download v15 WASM Artifacts 📥 - uses: actions/download-artifact@v4 - with: - name: wasm-artifacts-v15 - path: versions/15/wasm/ - - - name: Download v16 WASM Artifacts 📥 - uses: actions/download-artifact@v4 - with: - name: wasm-artifacts-v16 - path: versions/16/wasm/ - - - name: Download v17 WASM Artifacts 📥 - uses: actions/download-artifact@v4 - with: - name: wasm-artifacts-v17 - path: versions/17/wasm/ - - - name: Download v18 WASM Artifacts 📥 - uses: actions/download-artifact@v4 - with: - name: wasm-artifacts-v18 - path: versions/18/wasm/ - - - name: Build Types Packages 🏗 - run: | - for version in 13 14 15 16 17 18; do - echo "Building types for v${version}..." - cd types/${version} - pnpm run build - cd ../.. - done - - - name: Build Parser 🏗 - run: pnpm run build - working-directory: parser - - - name: Upload Parser Artifacts 📦 - uses: actions/upload-artifact@v4 - with: - name: parser-artifacts - path: parser/wasm/ - retention-days: 1 - - test-parser: - name: Test Parser on ${{ matrix.os }} ${{ matrix.os == 'ubuntu-latest' && '🐧' || matrix.os == 'macos-latest' && '🍎' || '🪟' }} - needs: build-parser - strategy: - matrix: - os: [ubuntu-latest, macos-latest, windows-latest] - fail-fast: false - runs-on: ${{ matrix.os }} - steps: - - name: Checkout Repository 📥 - uses: actions/checkout@v4 - - - name: Setup Node.js 🌐 - uses: actions/setup-node@v4 - with: - node-version: '20.x' - - - name: Setup pnpm 📦 - uses: pnpm/action-setup@v2 - with: - version: 8.15.1 - - - name: Get pnpm store directory 📁 - shell: bash - run: | - echo "STORE_PATH=$(pnpm store path --silent)" >> $GITHUB_ENV - - - name: Setup pnpm cache 🗄️ - uses: actions/cache@v3 - with: - path: ${{ env.STORE_PATH }} - key: ${{ runner.os }}-pnpm-store-${{ hashFiles('**/pnpm-lock.yaml') }} - restore-keys: | - ${{ runner.os }}-pnpm-store- - - - name: Install Dependencies 🧶 - run: pnpm install - - - name: Install Parser Dependencies 📦 - run: pnpm install - working-directory: parser - - - name: Download v13 WASM Artifacts 📥 - uses: actions/download-artifact@v4 - with: - name: wasm-artifacts-v13 - path: versions/13/wasm/ - - - name: Download v14 WASM Artifacts 📥 - uses: actions/download-artifact@v4 - with: - name: wasm-artifacts-v14 - path: versions/14/wasm/ - - - name: Download v15 WASM Artifacts 📥 - uses: actions/download-artifact@v4 - with: - name: wasm-artifacts-v15 - path: versions/15/wasm/ - - - name: Download v16 WASM Artifacts 📥 - uses: actions/download-artifact@v4 - with: - name: wasm-artifacts-v16 - path: versions/16/wasm/ - - - name: Download v17 WASM Artifacts 📥 - uses: actions/download-artifact@v4 - with: - name: wasm-artifacts-v17 - path: versions/17/wasm/ - - - name: Download v18 WASM Artifacts 📥 - uses: actions/download-artifact@v4 - with: - name: wasm-artifacts-v18 - path: versions/18/wasm/ - - - name: Download Parser Artifacts 📥 - uses: actions/download-artifact@v4 - with: - name: parser-artifacts - path: parser/wasm/ - - - name: Run Parser Tests 🔍 - run: pnpm run test - working-directory: parser From 85a8d72cef9d2e80c288496c8758a15bac71f5a1 Mon Sep 17 00:00:00 2001 From: Ben Asher Date: Sun, 28 Jun 2026 21:59:43 -0700 Subject: [PATCH 12/20] Add benchmark regression tracking in CI MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit New native-benchmark.yml runs benchmark/ci-bench.mjs on a fixed runner (ubuntu-24.04, under jemalloc) for every PR and push to main, tracking four smaller-is-better metrics against a gh-pages baseline: - large-query parse time (ms) - large-query peak RSS (MB) - large-query retained RSS (MB) - small-query latency (us/parse) Per-metric diffs are posted to the run summary every run (summary-always); a regression beyond 2x the baseline comments on the PR and fails the check (fail-on-alert, alert-threshold: 200%) — conservative to tolerate shared-runner noise. Baseline is written to gh-pages only on push to main; PRs compare only. Co-Authored-By: Claude Opus 4.8 --- .github/workflows/native-benchmark.yml | 68 +++++++++++++ native/README.md | 14 +++ native/benchmark/ci-bench.mjs | 128 +++++++++++++++++++++++++ 3 files changed, 210 insertions(+) create mode 100644 .github/workflows/native-benchmark.yml create mode 100644 native/benchmark/ci-bench.mjs diff --git a/.github/workflows/native-benchmark.yml b/.github/workflows/native-benchmark.yml new file mode 100644 index 0000000..0ec874a --- /dev/null +++ b/.github/workflows/native-benchmark.yml @@ -0,0 +1,68 @@ +name: Native Benchmark + +on: + push: + branches: [main] + paths: + - "native/**" + - ".github/workflows/native-benchmark.yml" + pull_request: + paths: + - "native/**" + - ".github/workflows/native-benchmark.yml" + +# Cancel superseded runs on the same ref so baselines aren't raced. +concurrency: + group: native-benchmark-${{ github.ref }} + cancel-in-progress: true + +permissions: + contents: write # push the baseline to gh-pages (on main) + pull-requests: write # comment on PRs when a regression alerts + +jobs: + benchmark: + name: Benchmark (linux-x64, jemalloc) + # Pinned to a single runner so stored baselines stay comparable. + runs-on: ubuntu-24.04 + steps: + - uses: actions/checkout@v7 + + - uses: actions/setup-node@v6 + with: + node-version: "24" + package-manager-cache: false + + - name: Install jemalloc + run: sudo apt-get update && sudo apt-get install -y libjemalloc2 + + - name: Build native addon + working-directory: native + run: | + npm install + make build PLATFORM_KEY=linux-x64 + npx tsc + + - name: Run benchmark (under jemalloc) + working-directory: native + env: + LD_PRELOAD: /usr/lib/x86_64-linux-gnu/libjemalloc.so.2 + run: node --expose-gc benchmark/ci-bench.mjs --out "$GITHUB_WORKSPACE/benchmark-results.json" + + - name: Track results and compare against baseline + uses: benchmark-action/github-action-benchmark@v1 + with: + name: native libpg-query (linux-x64, jemalloc) + tool: customSmallerIsBetter + output-file-path: benchmark-results.json + github-token: ${{ secrets.GITHUB_TOKEN }} + # Store/track history on gh-pages, but only write/push from main. + gh-pages-branch: gh-pages + benchmark-data-dir-path: dev/bench/native + auto-push: ${{ github.event_name == 'push' }} + save-data-file: ${{ github.event_name == 'push' }} + # Report every run; fail only on a large (2x) regression. + summary-always: true + comment-on-alert: true + fail-on-alert: true + alert-threshold: "200%" diff --git a/native/README.md b/native/README.md index 4427aa3..f3eaab0 100644 --- a/native/README.md +++ b/native/README.md @@ -148,8 +148,22 @@ node --expose-gc benchmark/memory.mjs --throughput # Compare against WASM (requires @libpg-query/parser installed) node --expose-gc benchmark/memory.mjs --all + +# CI regression benchmark (emits github-action-benchmark JSON) +node --expose-gc benchmark/ci-bench.mjs --out results.json ``` +### Regression tracking in CI + +The `Native Benchmark` workflow runs `ci-bench.mjs` on a fixed runner +(ubuntu-24.04, under jemalloc) for every PR and push to `main`. It tracks four +smaller-is-better metrics — large-query parse time, peak RSS, retained RSS, and +small-query latency — against a baseline stored on the `gh-pages` branch. Each +run posts the per-metric difference to the job summary; a regression beyond +**2× the baseline** comments on the PR and fails the check. The threshold is +deliberately conservative (`alert-threshold: 200%`) to tolerate shared-runner +noise — tune it in `.github/workflows/native-benchmark.yml`. + ## Building from source ```bash diff --git a/native/benchmark/ci-bench.mjs b/native/benchmark/ci-bench.mjs new file mode 100644 index 0000000..3e4edcb --- /dev/null +++ b/native/benchmark/ci-bench.mjs @@ -0,0 +1,128 @@ +/** + * CI regression benchmark. + * + * Emits results in the github-action-benchmark `customSmallerIsBetter` format + * so the difference vs. the stored baseline is reported on every PR and a + * conservative threshold can fail the build on a real regression. + * + * All metrics are normalized to "smaller is better": + * - large-query parse time (ms) + * - large-query peak RSS (MB) + * - large-query retained RSS after free (MB) + * - small-query latency (µs per parse) [inverse of throughput] + * + * Run with: node --expose-gc benchmark/ci-bench.mjs --out results.json + * For meaningful, stable memory numbers, preload jemalloc (see workflow). + */ + +import { argv } from "process"; +import { writeFileSync } from "fs"; +import { createRequire } from "module"; + +const require = createRequire(import.meta.url); +const MB = 1024 * 1024; + +function rss() { + return process.memoryUsage.rss(); +} + +function generateBigQuery(unions = 1500, cols = 400) { + const colList = Array.from({ length: cols }, (_, i) => `c${i}`).join(", "); + const select = `SELECT ${colList} FROM t`; + return Array.from({ length: unions }, () => select).join(" UNION ALL "); +} + +async function gcSettle(ms = 300) { + if (global.gc) global.gc(); + await new Promise((r) => setTimeout(r, ms)); + if (global.gc) global.gc(); + await new Promise((r) => setTimeout(r, ms)); +} + +function detectAllocator() { + const preload = + process.env.LD_PRELOAD || process.env.DYLD_INSERT_LIBRARIES || ""; + if (preload.includes("jemalloc")) return "jemalloc"; + if (preload.includes("mimalloc")) return "mimalloc"; + if (preload.includes("tcmalloc")) return "tcmalloc"; + return "system"; +} + +async function main() { + const outIdx = argv.indexOf("--out"); + const outFile = outIdx !== -1 ? argv[outIdx + 1] : "benchmark-results.json"; + + const lib = require("../dist/index.js"); + const allocator = detectAllocator(); + + const bigQuery = generateBigQuery(); + const smallQuery = "SELECT id, name FROM users WHERE id = $1 AND active = true"; + + // Warm up + for (let i = 0; i < 5; i++) lib.parseSync(bigQuery.slice(0, 200)); + + // --- Large query: parse time, peak RSS, retained RSS (3 cycles) --- + await gcSettle(); + const idleRss = rss(); + let peakRss = 0; + let totalParse = 0; + const cycles = 3; + for (let c = 0; c < cycles; c++) { + const start = performance.now(); + let result = lib.parseSync(bigQuery); + totalParse += performance.now() - start; + const cur = rss(); + if (cur > peakRss) peakRss = cur; + result = null; + await gcSettle(); + } + await gcSettle(); + const retainedRss = rss() - idleRss; + const avgParseMs = totalParse / cycles; + + // --- Small query: throughput -> µs/parse --- + const iterations = 20000; + for (let i = 0; i < 1000; i++) lib.parseSync(smallQuery); // warm + const tStart = performance.now(); + for (let i = 0; i < iterations; i++) lib.parseSync(smallQuery); + const elapsedMs = performance.now() - tStart; + const usPerParse = (elapsedMs * 1000) / iterations; + + const results = [ + { + name: "Large query parse time", + unit: "ms", + value: round(avgParseMs), + }, + { + name: "Large query peak RSS", + unit: "MB", + value: round(peakRss / MB), + }, + { + name: "Large query retained RSS", + unit: "MB", + value: round(retainedRss / MB), + }, + { + name: "Small query latency", + unit: "us/parse", + value: round(usPerParse, 3), + }, + ]; + + console.log(`Allocator: ${allocator}`); + console.table(results); + writeFileSync(outFile, JSON.stringify(results, null, 2) + "\n"); + console.log(`Wrote ${outFile}`); +} + +function round(n, dp = 1) { + const f = 10 ** dp; + return Math.round(n * f) / f; +} + +main().catch((e) => { + console.error(e); + process.exit(1); +}); From 5bbd5301a040bb54d8c17587ed37c403fb9df5b6 Mon Sep 17 00:00:00 2001 From: Ben Asher Date: Sun, 28 Jun 2026 22:09:40 -0700 Subject: [PATCH 13/20] README: clarify this fork builds only the native Node addon MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add a prominent note at the top of the root README stating that this fork builds, tests, and publishes only @ashbyhq/libpg-query-native (a native N-API addon for Node.js, PG 18) — no WASM/browser/Deno/Worker builds and no WASM build in CI. The upstream WASM packages remain in-tree for reference but are not built or published here; point readers to native/README.md. Co-Authored-By: Claude Opus 4.8 --- README.md | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/README.md b/README.md index 21f928f..32a03f4 100644 --- a/README.md +++ b/README.md @@ -15,6 +15,24 @@

+> [!IMPORTANT] +> **This fork (`ashbyhq`) builds only the native N-API addon, for Node.js.** +> +> CI builds, tests, and publishes a single package — [`@ashbyhq/libpg-query-native`](native/) — +> a native Node.js addon (PostgreSQL 18). There are **no WASM, browser, Deno, or +> Worker builds**, and there is no WASM build in CI. +> +> The WASM packages documented below (`libpg-query`, `@pgsql/parser`, etc.) remain +> in the tree for reference and upstream parity, but **this repository does not build +> or publish them**. For the native addon — install, API, platform support, and the +> native-vs-WASM memory comparison — see **[`native/README.md`](native/README.md)**. +> +> ```bash +> npm install @ashbyhq/libpg-query-native +> ``` +> +> The original upstream WASM documentation follows. + # The Real PostgreSQL Parser for JavaScript ### Bring the power of PostgreSQL's native parser to your TypeScript projects — no native builds, no platform headaches. From d71dddac2b3f02816b0d8b3189a84fa2cbae99bb Mon Sep 17 00:00:00 2001 From: Ben Asher Date: Sun, 28 Jun 2026 22:20:49 -0700 Subject: [PATCH 14/20] Pin TypeScript and run tsc via npm script, not npx - Pin typescript to exact 5.9.3 in native devDependencies (was ^5.3.3) so the compiler version is deterministic across local and CI builds. - Replace every `npx tsc` in the workflows with `npm run build:ts`, which runs the pinned local binary from node_modules/.bin (no on-the-fly npx fetch). Audited the repo: these were the only npx usages; none remain. Co-Authored-By: Claude Opus 4.8 --- .github/workflows/native-benchmark.yml | 2 +- .github/workflows/native-build.yml | 4 ++-- .github/workflows/native-ci.yml | 4 ++-- .github/workflows/native-release.yml | 2 +- native/package.json | 2 +- 5 files changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/native-benchmark.yml b/.github/workflows/native-benchmark.yml index 0ec874a..8634313 100644 --- a/.github/workflows/native-benchmark.yml +++ b/.github/workflows/native-benchmark.yml @@ -41,7 +41,7 @@ jobs: run: | npm install make build PLATFORM_KEY=linux-x64 - npx tsc + npm run build:ts - name: Run benchmark (under jemalloc) working-directory: native diff --git a/.github/workflows/native-build.yml b/.github/workflows/native-build.yml index e42aa67..d642aa5 100644 --- a/.github/workflows/native-build.yml +++ b/.github/workflows/native-build.yml @@ -56,7 +56,7 @@ jobs: run: | npm install make build PLATFORM_KEY=${{ matrix.platform }} - npx tsc + npm run build:ts npm test - name: Build and test (musl, in Alpine container) @@ -67,7 +67,7 @@ jobs: apk add --no-cache git make g++ python3 && npm install && make build PLATFORM_KEY=${{ matrix.platform }} && - npx tsc && + npm run build:ts && npm test ' diff --git a/.github/workflows/native-ci.yml b/.github/workflows/native-ci.yml index f86ce06..8a39e43 100644 --- a/.github/workflows/native-ci.yml +++ b/.github/workflows/native-ci.yml @@ -76,7 +76,7 @@ jobs: run: | cd native npm install - npx tsc + npm run build:ts node scripts/package-platforms.mjs mkdir -p dist-packs npm pack --pack-destination "$PWD/dist-packs" @@ -94,7 +94,7 @@ jobs: docker run --rm -v "$GITHUB_WORKSPACE/native:/work" -w /work \ "${{ matrix.container }}" sh -c ' apk add --no-cache git && - npm install && npx tsc && node scripts/package-platforms.mjs && + npm install && npm run build:ts && node scripts/package-platforms.mjs && mkdir -p dist-packs && npm pack --pack-destination /work/dist-packs && (cd packages/libpg-query-native-${{ matrix.platform }} && npm pack --pack-destination /work/dist-packs) && diff --git a/.github/workflows/native-release.yml b/.github/workflows/native-release.yml index 7e0411b..abd5f31 100644 --- a/.github/workflows/native-release.yml +++ b/.github/workflows/native-release.yml @@ -65,7 +65,7 @@ jobs: - name: Build TypeScript working-directory: native - run: npx tsc + run: npm run build:ts - name: Generate platform packages working-directory: native diff --git a/native/package.json b/native/package.json index 299dd92..a580124 100644 --- a/native/package.json +++ b/native/package.json @@ -45,7 +45,7 @@ "@types/node": "^26.0.1", "node-addon-api": "^8.3.1", "node-api-headers": "^1.5.0", - "typescript": "^5.3.3" + "typescript": "5.9.3" }, "optionalDependencies": { "@ashbyhq/libpg-query-native-darwin-arm64": "0.1.0", From 4ca5644612590fd5ac6fdc4634ced9e582c3ef2b Mon Sep 17 00:00:00 2001 From: Jeff Lubetkin Date: Mon, 29 Jun 2026 16:35:36 -0700 Subject: [PATCH 15/20] native: detect libc via process.report, not `ldd` MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit isMusl() shelled out to `ldd --version` and treated any failure as musl. Minimal images (distroless/scratch) ship no `ldd`, so the call throws and a glibc host is misdetected as musl — selecting a platform package/prebuild that doesn't exist and failing at load with "Native addon not found for linux-x64-musl". Use Node's process.report.header.glibcVersionRuntime (set on glibc, absent on musl), with a /lib/ld-musl-* fallback and a glibc default. Verified: distroless debian13 now resolves linux-x64; node:alpine still resolves musl. Bump to 0.1.1. Co-Authored-By: Claude Opus 4.8 (1M context) --- native/package.json | 12 ++++++------ native/src/index.ts | 24 +++++++++++++++++++----- 2 files changed, 25 insertions(+), 11 deletions(-) diff --git a/native/package.json b/native/package.json index a580124..ed96e82 100644 --- a/native/package.json +++ b/native/package.json @@ -1,6 +1,6 @@ { "name": "@ashbyhq/libpg-query-native", - "version": "0.1.0", + "version": "0.1.1", "description": "PostgreSQL query parser — native N-API + jemalloc backend", "main": "./dist/index.js", "types": "./dist/index.d.ts", @@ -48,10 +48,10 @@ "typescript": "5.9.3" }, "optionalDependencies": { - "@ashbyhq/libpg-query-native-darwin-arm64": "0.1.0", - "@ashbyhq/libpg-query-native-linux-x64": "0.1.0", - "@ashbyhq/libpg-query-native-linux-arm64": "0.1.0", - "@ashbyhq/libpg-query-native-linux-x64-musl": "0.1.0", - "@ashbyhq/libpg-query-native-linux-arm64-musl": "0.1.0" + "@ashbyhq/libpg-query-native-darwin-arm64": "0.1.1", + "@ashbyhq/libpg-query-native-linux-x64": "0.1.1", + "@ashbyhq/libpg-query-native-linux-arm64": "0.1.1", + "@ashbyhq/libpg-query-native-linux-x64-musl": "0.1.1", + "@ashbyhq/libpg-query-native-linux-arm64-musl": "0.1.1" } } diff --git a/native/src/index.ts b/native/src/index.ts index 43bbf7f..f7994a7 100644 --- a/native/src/index.ts +++ b/native/src/index.ts @@ -83,13 +83,27 @@ function loadNativeAddon(): NativeAddon { function isMusl(): boolean { if (process.platform !== "linux") return false; + // Prefer Node's own libc marker: `glibcVersionRuntime` is set on glibc and + // absent on musl. This is reliable in minimal images (distroless/scratch) + // that ship no `ldd` binary — shelling out there throws and would otherwise + // be misread as musl, selecting the wrong platform package. try { - const { execSync } = require("child_process"); - const ldd = execSync("ldd --version 2>&1", { encoding: "utf8" }); - return ldd.includes("musl"); + const report = process.report?.getReport?.() as + | { header?: { glibcVersionRuntime?: string } } + | undefined; + if (report?.header != null) { + return report.header.glibcVersionRuntime == null; + } } catch { - // ldd --version exits non-zero on musl - return true; + // fall through to the filesystem probe + } + // Fallback (older runtimes without process.report): look for the musl + // dynamic loader on disk. Default to glibc — the common case — when unsure. + try { + const fs = require("fs"); + return fs.readdirSync("/lib").some((f: string) => f.startsWith("ld-musl-")); + } catch { + return false; } } From 5302fd89edb9295fd0027b704ee3e8cb5bb3f059 Mon Sep 17 00:00:00 2001 From: Ben Asher Date: Tue, 30 Jun 2026 09:19:19 -0700 Subject: [PATCH 16/20] Set publishConfig.access=public on the main package The release workflow already passes `npm publish --access public`, but the main package.json lacked publishConfig, so a manual `npm publish` would default to restricted for a scoped (@ashbyhq) package. Add publishConfig.access=public to match the generated platform packages and make scoped public publishing the default regardless of how it's invoked. Co-Authored-By: Claude Opus 4.8 --- native/package.json | 3 +++ 1 file changed, 3 insertions(+) diff --git a/native/package.json b/native/package.json index ed96e82..dd970e7 100644 --- a/native/package.json +++ b/native/package.json @@ -15,6 +15,9 @@ "dist/**/*", "platforms.json" ], + "publishConfig": { + "access": "public" + }, "scripts": { "build:native": "make build", "build:ts": "tsc", From 0304aa31e863207badac3a506891f56951c9f1e4 Mon Sep 17 00:00:00 2001 From: Ben Asher Date: Tue, 30 Jun 2026 09:25:34 -0700 Subject: [PATCH 17/20] Support beta / prerelease publishes via npm dist-tags The release workflow published everything to the 'latest' dist-tag, so a prerelease (e.g. 0.2.0-beta.0) would have hijacked 'latest'. Add dist-tag support: - New optional `tag` dispatch input. - When blank, auto-derive: 'latest' for X.Y.Z, or the leading alpha prerelease id for X.Y.Z-.N (0.2.0-beta.1 -> beta, -rc -> rc, numeric prerelease -> beta). - Publish main + platform packages with `npm publish --access public --tag $NPM_TAG`. - Guard: refuse to publish a prerelease onto 'latest'. - Dry-run summary prints the resolved dist-tag. Derivation uses pure shell case/parameter-expansion (no grep/sed) for portability. Co-Authored-By: Claude Opus 4.8 --- .github/workflows/native-release.yml | 42 ++++++++++++++++++++++++---- 1 file changed, 37 insertions(+), 5 deletions(-) diff --git a/.github/workflows/native-release.yml b/.github/workflows/native-release.yml index abd5f31..ead5657 100644 --- a/.github/workflows/native-release.yml +++ b/.github/workflows/native-release.yml @@ -4,8 +4,12 @@ on: workflow_dispatch: inputs: version: - description: "Version to publish (e.g. 0.1.0)" + description: "Version to publish (e.g. 0.1.0 or 0.2.0-beta.0)" required: true + tag: + description: "npm dist-tag (blank = auto: 'latest' for releases, prerelease id for X.Y.Z-.N, e.g. beta)" + required: false + default: "" dry-run: description: "Dry run (skip npm publish)" type: boolean @@ -59,6 +63,34 @@ jobs: node scripts/sync-optional-deps.mjs echo "Version set to ${{ inputs.version }}" + - name: Determine npm dist-tag + run: | + VERSION="${{ inputs.version }}" + TAG="${{ inputs.tag }}" + if [ -z "$TAG" ]; then + case "$VERSION" in + *-*) + # prerelease: use the leading alpha id after '-' (0.2.0-beta.1 -> beta) + rest=${VERSION#*-} + id=${rest%%[!A-Za-z]*} + TAG=${id:-beta} + ;; + *) + TAG=latest + ;; + esac + fi + # Safety: never push a prerelease onto the 'latest' tag. + case "$VERSION" in + *-*) + if [ "$TAG" = "latest" ]; then + echo "Refusing to publish prerelease $VERSION to 'latest'"; exit 1 + fi + ;; + esac + echo "NPM_TAG=$TAG" >> "$GITHUB_ENV" + echo "Publishing with dist-tag: $TAG" + - name: Install dependencies working-directory: native run: npm install @@ -78,9 +110,9 @@ jobs: NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }} run: | for pkg_dir in packages/*/; do - echo "Publishing $(basename $pkg_dir)..." + echo "Publishing $(basename $pkg_dir) (tag: $NPM_TAG)..." cd "$pkg_dir" - npm publish --access public + npm publish --access public --tag "$NPM_TAG" cd ../.. done @@ -89,7 +121,7 @@ jobs: working-directory: native env: NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }} - run: npm publish --access public + run: npm publish --access public --tag "$NPM_TAG" - name: Dry run summary if: ${{ inputs.dry-run }} @@ -97,7 +129,7 @@ jobs: run: | echo "=== DRY RUN ===" echo "" - echo "Would publish version ${{ inputs.version }}:" + echo "Would publish version ${{ inputs.version }} with dist-tag: $NPM_TAG" echo "" echo "Platform packages:" for pkg_dir in packages/*/; do From 2ba52ec665b377235282fdbbff15ac1393a0755b Mon Sep 17 00:00:00 2001 From: Ben Asher Date: Tue, 30 Jun 2026 09:44:37 -0700 Subject: [PATCH 18/20] Address CodeRabbit review feedback Code correctness: - addon.cc: escape control chars (<0x20) as \u00XX in EscapeJsonString so error payloads stay valid JSON (was emitting raw \b/\f/\x01 -> JSON.parse SyntaxError instead of SqlError). - test/smoke.mjs: make the runner async and await each case (top-level await); the two async tests were never awaited, so failures passed silently. - test/errors.test.js: add assert.fail("Expected error") to the one negative test missing it, so it can't pass vacuously. - benchmark/memory.mjs: exit non-zero when a requested backend fails to load instead of silently dropping it. Security hardening (zizmor): - Avoid template injection: pass matrix.platform/container and inputs.version/ tag through `env:` and reference shell vars in run/docker blocks instead of inlining ${{ }} expressions. - Add least-privilege permissions: contents: read on native-build.yml and native-ci.yml; move native-benchmark.yml permissions to the job. - Pin all actions to commit SHAs (checkout v7.0.0, setup-node v6.4.0, upload-artifact v7.0.1, download-artifact v8.0.1, github-action-benchmark v1.20.7). Maintainability: - native-ci.yml: drop the duplicated matrix-generation job; consume the build workflow's matrix via a new workflow_call output. - Makefile: clone libpg_query into a temp dir and mv into place on success so an interrupted clone can't poison the cache. - README: note the jemalloc path varies by distro/arch. Skipped: the "async API is synchronous" note (by design, matches the WASM API). Co-Authored-By: Claude Opus 4.8 --- .github/workflows/native-benchmark.yml | 13 ++++--- .github/workflows/native-build.yml | 28 ++++++++++----- .github/workflows/native-ci.yml | 48 ++++++++++---------------- .github/workflows/native-release.yml | 28 +++++++++------ native/Makefile | 8 +++-- native/README.md | 5 +++ native/benchmark/memory.mjs | 6 ++++ native/src/addon.cc | 14 +++++++- native/test/errors.test.js | 1 + native/test/smoke.mjs | 20 +++++------ 10 files changed, 103 insertions(+), 68 deletions(-) diff --git a/.github/workflows/native-benchmark.yml b/.github/workflows/native-benchmark.yml index 8634313..2456dab 100644 --- a/.github/workflows/native-benchmark.yml +++ b/.github/workflows/native-benchmark.yml @@ -16,19 +16,18 @@ concurrency: group: native-benchmark-${{ github.ref }} cancel-in-progress: true -permissions: - contents: write # push the baseline to gh-pages (on main) - pull-requests: write # comment on PRs when a regression alerts - jobs: benchmark: name: Benchmark (linux-x64, jemalloc) # Pinned to a single runner so stored baselines stay comparable. runs-on: ubuntu-24.04 + permissions: + contents: write # push the baseline to gh-pages (on main) + pull-requests: write # comment on PRs when a regression alerts steps: - - uses: actions/checkout@v7 + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 - - uses: actions/setup-node@v6 + - uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 with: node-version: "24" package-manager-cache: false @@ -50,7 +49,7 @@ jobs: run: node --expose-gc benchmark/ci-bench.mjs --out "$GITHUB_WORKSPACE/benchmark-results.json" - name: Track results and compare against baseline - uses: benchmark-action/github-action-benchmark@v1 + uses: benchmark-action/github-action-benchmark@4bdcce38c94cec68da58d012ac24b7b1155efe8b # v1.20.7 with: name: native libpg-query (linux-x64, jemalloc) tool: customSmallerIsBetter diff --git a/.github/workflows/native-build.yml b/.github/workflows/native-build.yml index d642aa5..c491177 100644 --- a/.github/workflows/native-build.yml +++ b/.github/workflows/native-build.yml @@ -6,6 +6,13 @@ on: upload-retention-days: type: number default: 7 + outputs: + matrix: + description: "JSON build matrix derived from native/platforms.json" + value: ${{ jobs.matrix.outputs.include }} + +permissions: + contents: read jobs: matrix: @@ -14,7 +21,7 @@ jobs: outputs: include: ${{ steps.gen.outputs.include }} steps: - - uses: actions/checkout@v7 + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 - id: gen run: | include=$(node -e " @@ -41,11 +48,11 @@ jobs: # can't run inside an Alpine container on arm64 runners. musl builds run via # `docker run` instead, with the host actions executing on the glibc host. steps: - - uses: actions/checkout@v7 + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 - name: Setup Node.js if: ${{ !matrix.musl }} - uses: actions/setup-node@v6 + uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 with: node-version: "24" package-manager-cache: false @@ -53,26 +60,31 @@ jobs: - name: Build and test (glibc / macOS) if: ${{ !matrix.musl }} working-directory: native + env: + PLATFORM_KEY: ${{ matrix.platform }} run: | npm install - make build PLATFORM_KEY=${{ matrix.platform }} + make build PLATFORM_KEY="$PLATFORM_KEY" npm run build:ts npm test - name: Build and test (musl, in Alpine container) if: ${{ matrix.musl }} + env: + PLATFORM_KEY: ${{ matrix.platform }} + CONTAINER: ${{ matrix.container }} run: | - docker run --rm -v "$GITHUB_WORKSPACE/native:/work" -w /work \ - "${{ matrix.container }}" sh -c ' + docker run --rm -e PLATFORM_KEY -v "$GITHUB_WORKSPACE/native:/work" -w /work \ + "$CONTAINER" sh -c ' apk add --no-cache git make g++ python3 && npm install && - make build PLATFORM_KEY=${{ matrix.platform }} && + make build PLATFORM_KEY="$PLATFORM_KEY" && npm run build:ts && npm test ' - name: Upload prebuild - uses: actions/upload-artifact@v7 + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: prebuild-${{ matrix.platform }} path: native/prebuilds/${{ matrix.platform }}/libpg_query_native.node diff --git a/.github/workflows/native-ci.yml b/.github/workflows/native-ci.yml index 8a39e43..76260bb 100644 --- a/.github/workflows/native-ci.yml +++ b/.github/workflows/native-ci.yml @@ -13,6 +13,9 @@ on: - ".github/workflows/native-ci.yml" - ".github/workflows/native-build.yml" +permissions: + contents: read + jobs: build: name: Build @@ -20,50 +23,30 @@ jobs: with: upload-retention-days: 3 - package-test-matrix: - name: Generate package test matrix - runs-on: ubuntu-24.04 - outputs: - include: ${{ steps.gen.outputs.include }} - steps: - - uses: actions/checkout@v7 - - id: gen - run: | - include=$(node -e " - const p = require('./native/platforms.json'); - const matrix = Object.entries(p).map(([key, v]) => ({ - platform: key, - runner: v.runner, - container: v.container || '', - musl: (v.libc || []).includes('musl'), - })); - console.log(JSON.stringify(matrix)); - ") - echo "include=${include}" >> "$GITHUB_OUTPUT" - package-test: name: Package test ${{ matrix.platform }} - needs: [build, package-test-matrix] + needs: build runs-on: ${{ matrix.runner }} strategy: fail-fast: false + # Reuse the matrix the build workflow already derived from platforms.json. matrix: - include: ${{ fromJSON(needs.package-test-matrix.outputs.include) }} + include: ${{ fromJSON(needs.build.outputs.matrix) }} # No job-level `container` (JS actions can't run in an Alpine container on # arm64). The prebuild is downloaded on the glibc host; the musl smoke test # runs in the Alpine image via `docker run`. steps: - - uses: actions/checkout@v7 + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 - name: Setup Node.js if: ${{ !matrix.musl }} - uses: actions/setup-node@v6 + uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 with: node-version: "24" package-manager-cache: false - name: Download prebuild - uses: actions/download-artifact@v8 + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: name: prebuild-${{ matrix.platform }} path: native/prebuilds/${{ matrix.platform }} @@ -73,6 +56,8 @@ jobs: # lookups for the (unpublished) platform optionalDependencies. - name: Pack, install, smoke test (glibc / macOS) if: ${{ !matrix.musl }} + env: + PLATFORM_KEY: ${{ matrix.platform }} run: | cd native npm install @@ -80,7 +65,7 @@ jobs: node scripts/package-platforms.mjs mkdir -p dist-packs npm pack --pack-destination "$PWD/dist-packs" - (cd "packages/libpg-query-native-${{ matrix.platform }}" \ + (cd "packages/libpg-query-native-$PLATFORM_KEY" \ && npm pack --pack-destination "$PWD/../../dist-packs") mkdir -p /tmp/smoke-test && cd /tmp/smoke-test && npm init -y >/dev/null npm install --omit=optional "$GITHUB_WORKSPACE/native/dist-packs/"*.tgz @@ -90,14 +75,17 @@ jobs: - name: Pack, install, smoke test (musl, in Alpine container) if: ${{ matrix.musl }} + env: + PLATFORM_KEY: ${{ matrix.platform }} + CONTAINER: ${{ matrix.container }} run: | - docker run --rm -v "$GITHUB_WORKSPACE/native:/work" -w /work \ - "${{ matrix.container }}" sh -c ' + docker run --rm -e PLATFORM_KEY -v "$GITHUB_WORKSPACE/native:/work" -w /work \ + "$CONTAINER" sh -c ' apk add --no-cache git && npm install && npm run build:ts && node scripts/package-platforms.mjs && mkdir -p dist-packs && npm pack --pack-destination /work/dist-packs && - (cd packages/libpg-query-native-${{ matrix.platform }} && npm pack --pack-destination /work/dist-packs) && + (cd "packages/libpg-query-native-$PLATFORM_KEY" && npm pack --pack-destination /work/dist-packs) && mkdir -p /tmp/smoke && cd /tmp/smoke && npm init -y >/dev/null && npm install --omit=optional /work/dist-packs/*.tgz && echo "Installed @ashbyhq packages:" && ls node_modules/@ashbyhq && diff --git a/.github/workflows/native-release.yml b/.github/workflows/native-release.yml index ead5657..3dc544b 100644 --- a/.github/workflows/native-release.yml +++ b/.github/workflows/native-release.yml @@ -29,16 +29,16 @@ jobs: permissions: contents: write steps: - - uses: actions/checkout@v7 + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 - - uses: actions/setup-node@v6 + - uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 with: node-version: "24" package-manager-cache: false registry-url: "https://registry.npmjs.org" - name: Download all prebuilds - uses: actions/download-artifact@v8 + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: path: native/prebuilds pattern: prebuild-* @@ -58,15 +58,19 @@ jobs: - name: Set version working-directory: native + env: + VERSION: ${{ inputs.version }} run: | - npm version "${{ inputs.version }}" --no-git-tag-version + npm version "$VERSION" --no-git-tag-version node scripts/sync-optional-deps.mjs - echo "Version set to ${{ inputs.version }}" + echo "Version set to $VERSION" - name: Determine npm dist-tag + env: + VERSION: ${{ inputs.version }} + TAG_INPUT: ${{ inputs.tag }} run: | - VERSION="${{ inputs.version }}" - TAG="${{ inputs.tag }}" + TAG="$TAG_INPUT" if [ -z "$TAG" ]; then case "$VERSION" in *-*) @@ -126,10 +130,12 @@ jobs: - name: Dry run summary if: ${{ inputs.dry-run }} working-directory: native + env: + VERSION: ${{ inputs.version }} run: | echo "=== DRY RUN ===" echo "" - echo "Would publish version ${{ inputs.version }} with dist-tag: $NPM_TAG" + echo "Would publish version $VERSION with dist-tag: $NPM_TAG" echo "" echo "Platform packages:" for pkg_dir in packages/*/; do @@ -143,6 +149,8 @@ jobs: - name: Create git tag if: ${{ !inputs.dry-run }} + env: + VERSION: ${{ inputs.version }} run: | - git tag "native-v${{ inputs.version }}" - git push origin "native-v${{ inputs.version }}" + git tag "native-v$VERSION" + git push origin "native-v$VERSION" diff --git a/native/Makefile b/native/Makefile index 57479c9..6417aad 100644 --- a/native/Makefile +++ b/native/Makefile @@ -59,10 +59,14 @@ INCLUDES := \ -I$(NODE_ADDON_API_DIR) \ $(if $(NODE_INCLUDE),-I$(NODE_INCLUDE),) -# Clone and build libpg_query +# Clone and build libpg_query. Clone into a temp dir and move into place only on +# success, so an interrupted clone can't leave a poisoned (partial) cache dir +# that later builds would mistake for a complete checkout. $(LIBPG_QUERY_DIR): mkdir -p $(CACHE_DIR) - git clone -b $(LIBPG_QUERY_TAG) --single-branch --depth 1 $(LIBPG_QUERY_REPO) $(LIBPG_QUERY_DIR) + rm -rf $(LIBPG_QUERY_DIR).tmp + git clone -b $(LIBPG_QUERY_TAG) --single-branch --depth 1 $(LIBPG_QUERY_REPO) $(LIBPG_QUERY_DIR).tmp + mv $(LIBPG_QUERY_DIR).tmp $(LIBPG_QUERY_DIR) $(LIBPG_QUERY_LIB): $(LIBPG_QUERY_DIR) cd $(LIBPG_QUERY_DIR) && $(MAKE) build diff --git a/native/README.md b/native/README.md index f3eaab0..220f199 100644 --- a/native/README.md +++ b/native/README.md @@ -131,6 +131,11 @@ RUN apt-get install -y libjemalloc2 ENV LD_PRELOAD=/usr/lib/x86_64-linux-gnu/libjemalloc.so.2 ``` +> The `libjemalloc.so.2` path above is for Debian/Ubuntu on x86_64. It differs by +> distro and architecture (e.g. `/usr/lib/aarch64-linux-gnu/libjemalloc.so.2` on +> arm64, `/usr/lib64/libjemalloc.so.2` on RHEL/Fedora). Find it with +> `ldconfig -p | grep jemalloc`. + ## Benchmarks ```bash diff --git a/native/benchmark/memory.mjs b/native/benchmark/memory.mjs index 72d7d13..773ccb6 100644 --- a/native/benchmark/memory.mjs +++ b/native/benchmark/memory.mjs @@ -195,6 +195,10 @@ async function main() { if (throughput) { await benchmarkThroughput(native, label); } + } else { + // A requested backend failed to load — fail loudly rather than + // reporting a "successful" run that benchmarked nothing. + process.exitCode = 1; } } @@ -205,6 +209,8 @@ async function main() { if (throughput) { await benchmarkThroughput(wasm, "WASM (emscripten)"); } + } else { + process.exitCode = 1; } } diff --git a/native/src/addon.cc b/native/src/addon.cc index 5574619..0914f84 100644 --- a/native/src/addon.cc +++ b/native/src/addon.cc @@ -1,4 +1,5 @@ #include +#include #include extern "C" { @@ -16,7 +17,18 @@ static std::string EscapeJsonString(const std::string &s) { case '\n': out += "\\n"; break; case '\r': out += "\\r"; break; case '\t': out += "\\t"; break; - default: out += c; + case '\b': out += "\\b"; break; + case '\f': out += "\\f"; break; + default: + // Escape any remaining control characters (<0x20) as \u00XX so the + // payload stays valid JSON for JSON.parse() on the JS side. + if (static_cast(c) < 0x20) { + char buf[7]; + std::snprintf(buf, sizeof(buf), "\\u%04x", static_cast(c)); + out += buf; + } else { + out += c; + } } } return out; diff --git a/native/test/errors.test.js b/native/test/errors.test.js index dfc610a..828d3a6 100644 --- a/native/test/errors.test.js +++ b/native/test/errors.test.js @@ -7,6 +7,7 @@ describe("Error Handling", () => { it("should include sqlDetails property on parse errors", () => { try { parseSync("SELECT * FROM users WHERE id = @"); + assert.fail("Expected error"); } catch (error) { assert.ok("sqlDetails" in error); assert.ok("message" in error.sqlDetails); diff --git a/native/test/smoke.mjs b/native/test/smoke.mjs index 05e692e..3bdd591 100644 --- a/native/test/smoke.mjs +++ b/native/test/smoke.mjs @@ -22,9 +22,9 @@ function assert(condition, message) { } } -function test(name, fn) { +async function test(name, fn) { try { - fn(); + await fn(); console.log(` ok ${name}`); } catch (e) { console.error(` FAIL ${name}: ${e.message}`); @@ -38,45 +38,45 @@ console.log(`Platform: ${process.platform}-${process.arch}`); console.log(`Node: ${process.version}`); console.log(""); -test("parseSync returns parse tree", () => { +await test("parseSync returns parse tree", () => { const result = lib.parseSync("SELECT 1"); assert(result.version > 0, "version should be positive"); assert(result.stmts.length === 1, "should have 1 statement"); assert(result.stmts[0].stmt.SelectStmt, "should have SelectStmt"); }); -test("parse (async) works", async () => { +await test("parse (async) works", async () => { const result = await lib.parse("SELECT 1"); assert(result.stmts.length === 1, "should have 1 statement"); }); -test("fingerprintSync returns hex string", () => { +await test("fingerprintSync returns hex string", () => { const fp = lib.fingerprintSync("SELECT 1"); assert(typeof fp === "string", "should be string"); assert(fp.length === 16, "should be 16 chars"); assert(/^[0-9a-f]+$/.test(fp), "should be hex"); }); -test("normalizeSync replaces constants", () => { +await test("normalizeSync replaces constants", () => { const result = lib.normalizeSync("SELECT 1, 'hello'"); assert(result.includes("$1"), "should have $1"); assert(!result.includes("hello"), "should not have literal"); }); -test("scanSync returns tokens", () => { +await test("scanSync returns tokens", () => { const result = lib.scanSync("SELECT id FROM users"); assert(result.tokens.length > 0, "should have tokens"); assert(result.tokens[0].text === "SELECT", "first token should be SELECT"); }); -test("parsePlPgSQLSync works", () => { +await test("parsePlPgSQLSync works", () => { const result = lib.parsePlPgSQLSync( "CREATE FUNCTION test() RETURNS void AS $$ BEGIN NULL; END; $$ LANGUAGE plpgsql" ); assert(result.plpgsql_funcs, "should have plpgsql_funcs"); }); -test("parseSync throws SqlError on bad SQL", () => { +await test("parseSync throws SqlError on bad SQL", () => { try { lib.parseSync("SELECTT"); assert(false, "should have thrown"); @@ -87,7 +87,7 @@ test("parseSync throws SqlError on bad SQL", () => { } }); -test("loadModule is a no-op", async () => { +await test("loadModule is a no-op", async () => { await lib.loadModule(); }); From 6316f19edc97dd8abab364e6316e6bb99889691d Mon Sep 17 00:00:00 2001 From: Ben Asher Date: Tue, 30 Jun 2026 09:55:44 -0700 Subject: [PATCH 19/20] Disable checkout credential persistence (CodeRabbit) Set persist-credentials: false on actions/checkout in the build, ci, and benchmark workflows. Those jobs only read repo files and run build/test/pack steps after checkout, so the GITHUB_TOKEN doesn't need to sit in local git config (zizmor artipacked). The benchmark action pushes via its explicit github-token input, not the checkout credentials, so auto-push still works. native-release.yml is intentionally left as-is: its publish job pushes the git tag and relies on the persisted checkout token. Co-Authored-By: Claude Opus 4.8 --- .github/workflows/native-benchmark.yml | 2 ++ .github/workflows/native-build.yml | 4 ++++ .github/workflows/native-ci.yml | 2 ++ 3 files changed, 8 insertions(+) diff --git a/.github/workflows/native-benchmark.yml b/.github/workflows/native-benchmark.yml index 2456dab..03b3136 100644 --- a/.github/workflows/native-benchmark.yml +++ b/.github/workflows/native-benchmark.yml @@ -26,6 +26,8 @@ jobs: pull-requests: write # comment on PRs when a regression alerts steps: - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + persist-credentials: false - uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 with: diff --git a/.github/workflows/native-build.yml b/.github/workflows/native-build.yml index c491177..382eda1 100644 --- a/.github/workflows/native-build.yml +++ b/.github/workflows/native-build.yml @@ -22,6 +22,8 @@ jobs: include: ${{ steps.gen.outputs.include }} steps: - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + persist-credentials: false - id: gen run: | include=$(node -e " @@ -49,6 +51,8 @@ jobs: # `docker run` instead, with the host actions executing on the glibc host. steps: - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + persist-credentials: false - name: Setup Node.js if: ${{ !matrix.musl }} diff --git a/.github/workflows/native-ci.yml b/.github/workflows/native-ci.yml index 76260bb..fe221a9 100644 --- a/.github/workflows/native-ci.yml +++ b/.github/workflows/native-ci.yml @@ -37,6 +37,8 @@ jobs: # runs in the Alpine image via `docker run`. steps: - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + persist-credentials: false - name: Setup Node.js if: ${{ !matrix.musl }} From 34fa0dffd9bb7f722046e403873453f0cf8a6c21 Mon Sep 17 00:00:00 2001 From: Jeff Lubetkin Date: Tue, 30 Jun 2026 12:42:18 -0700 Subject: [PATCH 20/20] native: bump version to 0.1.1-beta.0 Published all 6 packages (main + 5 platform binaries) to npm under the 'beta' dist-tag at this version. Co-Authored-By: Claude Opus 4.8 (1M context) --- native/package.json | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/native/package.json b/native/package.json index dd970e7..5f1bbc2 100644 --- a/native/package.json +++ b/native/package.json @@ -1,6 +1,6 @@ { "name": "@ashbyhq/libpg-query-native", - "version": "0.1.1", + "version": "0.1.1-beta.0", "description": "PostgreSQL query parser — native N-API + jemalloc backend", "main": "./dist/index.js", "types": "./dist/index.d.ts", @@ -51,10 +51,10 @@ "typescript": "5.9.3" }, "optionalDependencies": { - "@ashbyhq/libpg-query-native-darwin-arm64": "0.1.1", - "@ashbyhq/libpg-query-native-linux-x64": "0.1.1", - "@ashbyhq/libpg-query-native-linux-arm64": "0.1.1", - "@ashbyhq/libpg-query-native-linux-x64-musl": "0.1.1", - "@ashbyhq/libpg-query-native-linux-arm64-musl": "0.1.1" + "@ashbyhq/libpg-query-native-darwin-arm64": "0.1.1-beta.0", + "@ashbyhq/libpg-query-native-linux-x64": "0.1.1-beta.0", + "@ashbyhq/libpg-query-native-linux-arm64": "0.1.1-beta.0", + "@ashbyhq/libpg-query-native-linux-x64-musl": "0.1.1-beta.0", + "@ashbyhq/libpg-query-native-linux-arm64-musl": "0.1.1-beta.0" } }