diff --git a/.github/workflows/build-wasm-no-docker.yaml b/.github/workflows/build-wasm-no-docker.yaml deleted file mode 100644 index 71d2029..0000000 --- a/.github/workflows/build-wasm-no-docker.yaml +++ /dev/null @@ -1,56 +0,0 @@ -name: Build Wasm No Docker ๐Ÿ›  -'on': - workflow_dispatch: null -jobs: - build-wasm-no-docker: - runs-on: ubuntu-latest - steps: - - name: Checkout Repository ๐Ÿ“ฅ - uses: actions/checkout@v4 - - name: Setup Node.js ๐ŸŒ - uses: actions/setup-node@v4 - with: - node-version: 20.x - - - name: Setup pnpm ๐Ÿ“ฆ - uses: pnpm/action-setup@v2 - with: - version: 8.15.0 - - - name: Get pnpm store directory ๐Ÿ“ - shell: bash - run: | - echo "STORE_PATH=$(pnpm store path --silent)" >> $GITHUB_ENV - - - name: Setup pnpm cache ๐Ÿ—„๏ธ - uses: actions/cache@v3 - with: - path: ${{ env.STORE_PATH }} - key: ${{ runner.os }}-pnpm-store-${{ hashFiles('**/pnpm-lock.yaml') }} - restore-keys: | - ${{ runner.os }}-pnpm-store- - - - name: Install Dependencies ๐Ÿงถ - run: pnpm install - - - name: Install Emscripten โœ๐Ÿป - run: | - sudo apt-get update - sudo apt-get install cmake python3 python3-pip - git clone --branch 3.1.59 --depth 1 https://github.com/emscripten-core/emsdk.git - cd emsdk - ./emsdk install 3.1.59 - ./emsdk activate 3.1.59 - source ./emsdk_env.sh - working-directory: full - - name: Build with Emscripten ๐Ÿ— - run: | - source ./emsdk/emsdk_env.sh - emmake make - emmake make build - working-directory: full - - name: Archive production artifacts ๐Ÿ› - uses: actions/upload-artifact@v4 - with: - name: wasm-artifacts - path: full/wasm diff --git a/.github/workflows/build-wasm.yml b/.github/workflows/build-wasm.yml deleted file mode 100644 index d520032..0000000 --- a/.github/workflows/build-wasm.yml +++ /dev/null @@ -1,48 +0,0 @@ -name: Build Wasm ๐Ÿ›  - -on: - workflow_dispatch: - -jobs: - build-wasm: - runs-on: macos-latest - steps: - - name: Checkout Repository ๐Ÿ“ฅ - uses: actions/checkout@v4 - - - name: Setup Node.js ๐ŸŒ - uses: actions/setup-node@v4 - with: - node-version: '20.x' - - - name: Setup pnpm ๐Ÿ“ฆ - uses: pnpm/action-setup@v2 - with: - version: 8.15.0 - - - name: Get pnpm store directory ๐Ÿ“ - shell: bash - run: | - echo "STORE_PATH=$(pnpm store path --silent)" >> $GITHUB_ENV - - - name: Setup pnpm cache ๐Ÿ—„๏ธ - uses: actions/cache@v3 - with: - path: ${{ env.STORE_PATH }} - key: ${{ runner.os }}-pnpm-store-${{ hashFiles('**/pnpm-lock.yaml') }} - restore-keys: | - ${{ runner.os }}-pnpm-store- - - - name: Install Dependencies ๐Ÿงถ - run: pnpm install - - - name: Build WASM ๐Ÿ— - run: pnpm run build - working-directory: full - - - name: Archive production artifacts ๐Ÿ› - uses: actions/upload-artifact@v4 - with: - name: wasm-artifacts - path: full/wasm/ - retention-days: 7 diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml deleted file mode 100644 index dac525b..0000000 --- a/.github/workflows/ci.yml +++ /dev/null @@ -1,310 +0,0 @@ -name: CI ๐Ÿš€ - -on: - pull_request: - types: [opened, synchronize, reopened] - push: - workflow_dispatch: - -jobs: - build-wasm: - name: Build WASM ${{ matrix.package }} ๐Ÿ”ง - runs-on: ubuntu-latest - strategy: - matrix: - package: - - { name: 'full', path: 'full', version: '18' } - - { name: 'v13', path: 'versions/13', version: '13' } - - { name: 'v14', path: 'versions/14', version: '14' } - - { name: 'v15', path: 'versions/15', version: '15' } - - { name: 'v16', path: 'versions/16', version: '16' } - - { name: 'v17', path: 'versions/17', version: '17' } - - { name: 'v18', path: 'versions/18', version: '18' } - fail-fast: false - steps: - - name: Checkout Repository ๐Ÿ“ฅ - uses: actions/checkout@v4 - - - name: Setup Node.js ๐ŸŒ - uses: actions/setup-node@v4 - with: - node-version: '20.x' - - - name: Setup pnpm ๐Ÿ“ฆ - uses: pnpm/action-setup@v2 - with: - version: 8.15.1 - - - name: Get pnpm store directory ๐Ÿ“ - shell: bash - run: | - echo "STORE_PATH=$(pnpm store path --silent)" >> $GITHUB_ENV - - - name: Setup pnpm cache ๐Ÿ—„๏ธ - uses: actions/cache@v3 - with: - path: ${{ env.STORE_PATH }} - key: ${{ runner.os }}-pnpm-store-${{ hashFiles('**/pnpm-lock.yaml') }} - restore-keys: | - ${{ runner.os }}-pnpm-store- - - - name: Install Dependencies ๐Ÿงถ - run: pnpm install - - - name: Build WASM ๐Ÿ— - run: | - if [ "${{ matrix.package.name }}" = "v13" ]; then - # Download prebuilt WASM for v13 since it can't build in CI - mkdir -p wasm - curl -o v13.tgz "https://registry.npmjs.org/@libpg-query/v13/-/v13-13.5.7.tgz" - tar -xzf v13.tgz --strip-components=1 package/wasm - rm v13.tgz - else - pnpm run build - fi - working-directory: ${{ matrix.package.path }} - - - name: Upload WASM Artifacts ๐Ÿ“ฆ - uses: actions/upload-artifact@v4 - with: - name: wasm-artifacts-${{ matrix.package.name }} - path: ${{ matrix.package.path }}/wasm/ - retention-days: 1 - - test: - name: Test ${{ matrix.package.name }} on ${{ matrix.os }} ${{ matrix.os == 'ubuntu-latest' && '๐Ÿง' || matrix.os == 'macos-latest' && '๐ŸŽ' || '๐ŸชŸ' }} - needs: build-wasm - strategy: - matrix: - os: [ubuntu-latest, macos-latest, windows-latest] - package: - - { name: 'full', path: 'full', version: '18' } - - { name: 'v13', path: 'versions/13', version: '13' } - - { name: 'v14', path: 'versions/14', version: '14' } - - { name: 'v15', path: 'versions/15', version: '15' } - - { name: 'v16', path: 'versions/16', version: '16' } - - { name: 'v17', path: 'versions/17', version: '17' } - - { name: 'v18', path: 'versions/18', version: '18' } - fail-fast: false - runs-on: ${{ matrix.os }} - steps: - - name: Checkout Repository ๐Ÿ“ฅ - uses: actions/checkout@v4 - - - name: Setup Node.js ๐ŸŒ - uses: actions/setup-node@v4 - with: - node-version: '20.x' - - - name: Setup pnpm ๐Ÿ“ฆ - uses: pnpm/action-setup@v2 - with: - version: 8.15.1 - - - name: Get pnpm store directory ๐Ÿ“ - shell: bash - run: | - echo "STORE_PATH=$(pnpm store path --silent)" >> $GITHUB_ENV - - - name: Setup pnpm cache ๐Ÿ—„๏ธ - uses: actions/cache@v3 - with: - path: ${{ env.STORE_PATH }} - key: ${{ runner.os }}-pnpm-store-${{ hashFiles('**/pnpm-lock.yaml') }} - restore-keys: | - ${{ runner.os }}-pnpm-store- - - - name: Install Dependencies ๐Ÿงถ - run: pnpm install - - - name: Download WASM Artifacts ๐Ÿ“ฅ - uses: actions/download-artifact@v4 - with: - name: wasm-artifacts-${{ matrix.package.name }} - path: ${{ matrix.package.path }}/wasm/ - - - name: Run Tests ๐Ÿ” - run: pnpm run test - working-directory: ${{ matrix.package.path }} - - build-parser: - name: Build Parser Package ๐Ÿ“ฆ - needs: build-wasm - runs-on: ubuntu-latest - steps: - - name: Checkout Repository ๐Ÿ“ฅ - uses: actions/checkout@v4 - - - name: Setup Node.js ๐ŸŒ - uses: actions/setup-node@v4 - with: - node-version: '20.x' - - - name: Setup pnpm ๐Ÿ“ฆ - uses: pnpm/action-setup@v2 - with: - version: 8.15.1 - - - name: Get pnpm store directory ๐Ÿ“ - shell: bash - run: | - echo "STORE_PATH=$(pnpm store path --silent)" >> $GITHUB_ENV - - - name: Setup pnpm cache ๐Ÿ—„๏ธ - uses: actions/cache@v3 - with: - path: ${{ env.STORE_PATH }} - key: ${{ runner.os }}-pnpm-store-${{ hashFiles('**/pnpm-lock.yaml') }} - restore-keys: | - ${{ runner.os }}-pnpm-store- - - - name: Install Dependencies ๐Ÿงถ - run: pnpm install - - - name: Install Parser Dependencies ๐Ÿ“ฆ - run: pnpm install - working-directory: parser - - - name: Download v13 WASM Artifacts ๐Ÿ“ฅ - uses: actions/download-artifact@v4 - with: - name: wasm-artifacts-v13 - path: versions/13/wasm/ - - - name: Download v14 WASM Artifacts ๐Ÿ“ฅ - uses: actions/download-artifact@v4 - with: - name: wasm-artifacts-v14 - path: versions/14/wasm/ - - - name: Download v15 WASM Artifacts ๐Ÿ“ฅ - uses: actions/download-artifact@v4 - with: - name: wasm-artifacts-v15 - path: versions/15/wasm/ - - - name: Download v16 WASM Artifacts ๐Ÿ“ฅ - uses: actions/download-artifact@v4 - with: - name: wasm-artifacts-v16 - path: versions/16/wasm/ - - - name: Download v17 WASM Artifacts ๐Ÿ“ฅ - uses: actions/download-artifact@v4 - with: - name: wasm-artifacts-v17 - path: versions/17/wasm/ - - - name: Download v18 WASM Artifacts ๐Ÿ“ฅ - uses: actions/download-artifact@v4 - with: - name: wasm-artifacts-v18 - path: versions/18/wasm/ - - - name: Build Types Packages ๐Ÿ— - run: | - for version in 13 14 15 16 17 18; do - echo "Building types for v${version}..." - cd types/${version} - pnpm run build - cd ../.. - done - - - name: Build Parser ๐Ÿ— - run: pnpm run build - working-directory: parser - - - name: Upload Parser Artifacts ๐Ÿ“ฆ - uses: actions/upload-artifact@v4 - with: - name: parser-artifacts - path: parser/wasm/ - retention-days: 1 - - test-parser: - name: Test Parser on ${{ matrix.os }} ${{ matrix.os == 'ubuntu-latest' && '๐Ÿง' || matrix.os == 'macos-latest' && '๐ŸŽ' || '๐ŸชŸ' }} - needs: build-parser - strategy: - matrix: - os: [ubuntu-latest, macos-latest, windows-latest] - fail-fast: false - runs-on: ${{ matrix.os }} - steps: - - name: Checkout Repository ๐Ÿ“ฅ - uses: actions/checkout@v4 - - - name: Setup Node.js ๐ŸŒ - uses: actions/setup-node@v4 - with: - node-version: '20.x' - - - name: Setup pnpm ๐Ÿ“ฆ - uses: pnpm/action-setup@v2 - with: - version: 8.15.1 - - - name: Get pnpm store directory ๐Ÿ“ - shell: bash - run: | - echo "STORE_PATH=$(pnpm store path --silent)" >> $GITHUB_ENV - - - name: Setup pnpm cache ๐Ÿ—„๏ธ - uses: actions/cache@v3 - with: - path: ${{ env.STORE_PATH }} - key: ${{ runner.os }}-pnpm-store-${{ hashFiles('**/pnpm-lock.yaml') }} - restore-keys: | - ${{ runner.os }}-pnpm-store- - - - name: Install Dependencies ๐Ÿงถ - run: pnpm install - - - name: Install Parser Dependencies ๐Ÿ“ฆ - run: pnpm install - working-directory: parser - - - name: Download v13 WASM Artifacts ๐Ÿ“ฅ - uses: actions/download-artifact@v4 - with: - name: wasm-artifacts-v13 - path: versions/13/wasm/ - - - name: Download v14 WASM Artifacts ๐Ÿ“ฅ - uses: actions/download-artifact@v4 - with: - name: wasm-artifacts-v14 - path: versions/14/wasm/ - - - name: Download v15 WASM Artifacts ๐Ÿ“ฅ - uses: actions/download-artifact@v4 - with: - name: wasm-artifacts-v15 - path: versions/15/wasm/ - - - name: Download v16 WASM Artifacts ๐Ÿ“ฅ - uses: actions/download-artifact@v4 - with: - name: wasm-artifacts-v16 - path: versions/16/wasm/ - - - name: Download v17 WASM Artifacts ๐Ÿ“ฅ - uses: actions/download-artifact@v4 - with: - name: wasm-artifacts-v17 - path: versions/17/wasm/ - - - name: Download v18 WASM Artifacts ๐Ÿ“ฅ - uses: actions/download-artifact@v4 - with: - name: wasm-artifacts-v18 - path: versions/18/wasm/ - - - name: Download Parser Artifacts ๐Ÿ“ฅ - uses: actions/download-artifact@v4 - with: - name: parser-artifacts - path: parser/wasm/ - - - name: Run Parser Tests ๐Ÿ” - run: pnpm run test - working-directory: parser diff --git a/.github/workflows/native-benchmark.yml b/.github/workflows/native-benchmark.yml new file mode 100644 index 0000000..03b3136 --- /dev/null +++ b/.github/workflows/native-benchmark.yml @@ -0,0 +1,69 @@ +name: Native Benchmark + +on: + push: + branches: [main] + paths: + - "native/**" + - ".github/workflows/native-benchmark.yml" + pull_request: + paths: + - "native/**" + - ".github/workflows/native-benchmark.yml" + +# Cancel superseded runs on the same ref so baselines aren't raced. +concurrency: + group: native-benchmark-${{ github.ref }} + cancel-in-progress: true + +jobs: + benchmark: + name: Benchmark (linux-x64, jemalloc) + # Pinned to a single runner so stored baselines stay comparable. + runs-on: ubuntu-24.04 + permissions: + contents: write # push the baseline to gh-pages (on main) + pull-requests: write # comment on PRs when a regression alerts + steps: + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + persist-credentials: false + + - uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 + with: + node-version: "24" + package-manager-cache: false + + - name: Install jemalloc + run: sudo apt-get update && sudo apt-get install -y libjemalloc2 + + - name: Build native addon + working-directory: native + run: | + npm install + make build PLATFORM_KEY=linux-x64 + npm run build:ts + + - name: Run benchmark (under jemalloc) + working-directory: native + env: + LD_PRELOAD: /usr/lib/x86_64-linux-gnu/libjemalloc.so.2 + run: node --expose-gc benchmark/ci-bench.mjs --out "$GITHUB_WORKSPACE/benchmark-results.json" + + - name: Track results and compare against baseline + uses: benchmark-action/github-action-benchmark@4bdcce38c94cec68da58d012ac24b7b1155efe8b # v1.20.7 + with: + name: native libpg-query (linux-x64, jemalloc) + tool: customSmallerIsBetter + output-file-path: benchmark-results.json + github-token: ${{ secrets.GITHUB_TOKEN }} + # Store/track history on gh-pages, but only write/push from main. + gh-pages-branch: gh-pages + benchmark-data-dir-path: dev/bench/native + auto-push: ${{ github.event_name == 'push' }} + save-data-file: ${{ github.event_name == 'push' }} + # Report every run; fail only on a large (2x) regression. + summary-always: true + comment-on-alert: true + fail-on-alert: true + alert-threshold: "200%" diff --git a/.github/workflows/native-build.yml b/.github/workflows/native-build.yml new file mode 100644 index 0000000..382eda1 --- /dev/null +++ b/.github/workflows/native-build.yml @@ -0,0 +1,95 @@ +name: Native Build (reusable) + +on: + workflow_call: + inputs: + upload-retention-days: + type: number + default: 7 + outputs: + matrix: + description: "JSON build matrix derived from native/platforms.json" + value: ${{ jobs.matrix.outputs.include }} + +permissions: + contents: read + +jobs: + matrix: + name: Generate matrix + runs-on: ubuntu-24.04 + outputs: + include: ${{ steps.gen.outputs.include }} + steps: + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + persist-credentials: false + - id: gen + run: | + include=$(node -e " + const p = require('./native/platforms.json'); + const matrix = Object.entries(p).map(([key, v]) => ({ + platform: key, + runner: v.runner, + container: v.container || '', + musl: (v.libc || []).includes('musl'), + })); + console.log(JSON.stringify(matrix)); + ") + echo "include=${include}" >> "$GITHUB_OUTPUT" + + build: + name: Build ${{ matrix.platform }} + needs: matrix + runs-on: ${{ matrix.runner }} + strategy: + fail-fast: false + matrix: + include: ${{ fromJSON(needs.matrix.outputs.include) }} + # No job-level `container`: JavaScript actions (checkout/setup-node/upload) + # can't run inside an Alpine container on arm64 runners. musl builds run via + # `docker run` instead, with the host actions executing on the glibc host. + steps: + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + persist-credentials: false + + - name: Setup Node.js + if: ${{ !matrix.musl }} + uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 + with: + node-version: "24" + package-manager-cache: false + + - name: Build and test (glibc / macOS) + if: ${{ !matrix.musl }} + working-directory: native + env: + PLATFORM_KEY: ${{ matrix.platform }} + run: | + npm install + make build PLATFORM_KEY="$PLATFORM_KEY" + npm run build:ts + npm test + + - name: Build and test (musl, in Alpine container) + if: ${{ matrix.musl }} + env: + PLATFORM_KEY: ${{ matrix.platform }} + CONTAINER: ${{ matrix.container }} + run: | + docker run --rm -e PLATFORM_KEY -v "$GITHUB_WORKSPACE/native:/work" -w /work \ + "$CONTAINER" sh -c ' + apk add --no-cache git make g++ python3 && + npm install && + make build PLATFORM_KEY="$PLATFORM_KEY" && + npm run build:ts && + npm test + ' + + - name: Upload prebuild + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: prebuild-${{ matrix.platform }} + path: native/prebuilds/${{ matrix.platform }}/libpg_query_native.node + retention-days: ${{ inputs.upload-retention-days }} diff --git a/.github/workflows/native-ci.yml b/.github/workflows/native-ci.yml new file mode 100644 index 0000000..fe221a9 --- /dev/null +++ b/.github/workflows/native-ci.yml @@ -0,0 +1,96 @@ +name: Native CI + +on: + push: + branches: [napi-jemalloc, main] + paths: + - "native/**" + - ".github/workflows/native-ci.yml" + - ".github/workflows/native-build.yml" + pull_request: + paths: + - "native/**" + - ".github/workflows/native-ci.yml" + - ".github/workflows/native-build.yml" + +permissions: + contents: read + +jobs: + build: + name: Build + uses: ./.github/workflows/native-build.yml + with: + upload-retention-days: 3 + + package-test: + name: Package test ${{ matrix.platform }} + needs: build + runs-on: ${{ matrix.runner }} + strategy: + fail-fast: false + # Reuse the matrix the build workflow already derived from platforms.json. + matrix: + include: ${{ fromJSON(needs.build.outputs.matrix) }} + # No job-level `container` (JS actions can't run in an Alpine container on + # arm64). The prebuild is downloaded on the glibc host; the musl smoke test + # runs in the Alpine image via `docker run`. + steps: + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + persist-credentials: false + + - name: Setup Node.js + if: ${{ !matrix.musl }} + uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 + with: + node-version: "24" + package-manager-cache: false + + - name: Download prebuild + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: prebuild-${{ matrix.platform }} + path: native/prebuilds/${{ matrix.platform }} + + # Pack the main + platform packages into tarballs and install them in an + # isolated dir. Tarballs + --omit=optional install cleanly without registry + # lookups for the (unpublished) platform optionalDependencies. + - name: Pack, install, smoke test (glibc / macOS) + if: ${{ !matrix.musl }} + env: + PLATFORM_KEY: ${{ matrix.platform }} + run: | + cd native + npm install + npm run build:ts + node scripts/package-platforms.mjs + mkdir -p dist-packs + npm pack --pack-destination "$PWD/dist-packs" + (cd "packages/libpg-query-native-$PLATFORM_KEY" \ + && npm pack --pack-destination "$PWD/../../dist-packs") + mkdir -p /tmp/smoke-test && cd /tmp/smoke-test && npm init -y >/dev/null + npm install --omit=optional "$GITHUB_WORKSPACE/native/dist-packs/"*.tgz + echo "Installed @ashbyhq packages:"; ls node_modules/@ashbyhq + cp "$GITHUB_WORKSPACE/native/test/smoke.mjs" smoke.mjs + node smoke.mjs + + - name: Pack, install, smoke test (musl, in Alpine container) + if: ${{ matrix.musl }} + env: + PLATFORM_KEY: ${{ matrix.platform }} + CONTAINER: ${{ matrix.container }} + run: | + docker run --rm -e PLATFORM_KEY -v "$GITHUB_WORKSPACE/native:/work" -w /work \ + "$CONTAINER" sh -c ' + apk add --no-cache git && + npm install && npm run build:ts && node scripts/package-platforms.mjs && + mkdir -p dist-packs && + npm pack --pack-destination /work/dist-packs && + (cd "packages/libpg-query-native-$PLATFORM_KEY" && npm pack --pack-destination /work/dist-packs) && + mkdir -p /tmp/smoke && cd /tmp/smoke && npm init -y >/dev/null && + npm install --omit=optional /work/dist-packs/*.tgz && + echo "Installed @ashbyhq packages:" && ls node_modules/@ashbyhq && + cp /work/test/smoke.mjs smoke.mjs && + node smoke.mjs + ' diff --git a/.github/workflows/native-release.yml b/.github/workflows/native-release.yml new file mode 100644 index 0000000..3dc544b --- /dev/null +++ b/.github/workflows/native-release.yml @@ -0,0 +1,156 @@ +name: Native Release + +on: + workflow_dispatch: + inputs: + version: + description: "Version to publish (e.g. 0.1.0 or 0.2.0-beta.0)" + required: true + tag: + description: "npm dist-tag (blank = auto: 'latest' for releases, prerelease id for X.Y.Z-.N, e.g. beta)" + required: false + default: "" + dry-run: + description: "Dry run (skip npm publish)" + type: boolean + default: true + +jobs: + build: + name: Build + uses: ./.github/workflows/native-build.yml + with: + upload-retention-days: 1 + + publish: + name: Publish to npm + needs: build + runs-on: ubuntu-24.04 + permissions: + contents: write + steps: + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + + - uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 + with: + node-version: "24" + package-manager-cache: false + registry-url: "https://registry.npmjs.org" + + - name: Download all prebuilds + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + path: native/prebuilds + pattern: prebuild-* + merge-multiple: false + + - name: Arrange prebuilds into expected layout + working-directory: native + run: | + for dir in prebuilds/prebuild-*/; do + platform=$(basename "$dir" | sed 's/prebuild-//') + mkdir -p "prebuilds/${platform}" + mv "${dir}libpg_query_native.node" "prebuilds/${platform}/" + rmdir "$dir" + done + echo "Prebuilds:" + find prebuilds -name '*.node' | sort + + - name: Set version + working-directory: native + env: + VERSION: ${{ inputs.version }} + run: | + npm version "$VERSION" --no-git-tag-version + node scripts/sync-optional-deps.mjs + echo "Version set to $VERSION" + + - name: Determine npm dist-tag + env: + VERSION: ${{ inputs.version }} + TAG_INPUT: ${{ inputs.tag }} + run: | + TAG="$TAG_INPUT" + if [ -z "$TAG" ]; then + case "$VERSION" in + *-*) + # prerelease: use the leading alpha id after '-' (0.2.0-beta.1 -> beta) + rest=${VERSION#*-} + id=${rest%%[!A-Za-z]*} + TAG=${id:-beta} + ;; + *) + TAG=latest + ;; + esac + fi + # Safety: never push a prerelease onto the 'latest' tag. + case "$VERSION" in + *-*) + if [ "$TAG" = "latest" ]; then + echo "Refusing to publish prerelease $VERSION to 'latest'"; exit 1 + fi + ;; + esac + echo "NPM_TAG=$TAG" >> "$GITHUB_ENV" + echo "Publishing with dist-tag: $TAG" + + - name: Install dependencies + working-directory: native + run: npm install + + - name: Build TypeScript + working-directory: native + run: npm run build:ts + + - name: Generate platform packages + working-directory: native + run: node scripts/package-platforms.mjs + + - name: Publish platform packages + if: ${{ !inputs.dry-run }} + working-directory: native + env: + NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }} + run: | + for pkg_dir in packages/*/; do + echo "Publishing $(basename $pkg_dir) (tag: $NPM_TAG)..." + cd "$pkg_dir" + npm publish --access public --tag "$NPM_TAG" + cd ../.. + done + + - name: Publish main package + if: ${{ !inputs.dry-run }} + working-directory: native + env: + NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }} + run: npm publish --access public --tag "$NPM_TAG" + + - name: Dry run summary + if: ${{ inputs.dry-run }} + working-directory: native + env: + VERSION: ${{ inputs.version }} + run: | + echo "=== DRY RUN ===" + echo "" + echo "Would publish version $VERSION with dist-tag: $NPM_TAG" + echo "" + echo "Platform packages:" + for pkg_dir in packages/*/; do + size=$(du -h "${pkg_dir}libpg_query_native.node" | cut -f1) + echo " $(basename $pkg_dir): ${size}" + done + echo "" + echo "Main package: @ashbyhq/libpg-query-native" + echo "" + echo "To publish for real, re-run with dry-run unchecked." + + - name: Create git tag + if: ${{ !inputs.dry-run }} + env: + VERSION: ${{ inputs.version }} + run: | + git tag "native-v$VERSION" + git push origin "native-v$VERSION" diff --git a/README.md b/README.md index 21f928f..32a03f4 100644 --- a/README.md +++ b/README.md @@ -15,6 +15,24 @@

+> [!IMPORTANT] +> **This fork (`ashbyhq`) builds only the native N-API addon, for Node.js.** +> +> CI builds, tests, and publishes a single package โ€” [`@ashbyhq/libpg-query-native`](native/) โ€” +> a native Node.js addon (PostgreSQL 18). There are **no WASM, browser, Deno, or +> Worker builds**, and there is no WASM build in CI. +> +> The WASM packages documented below (`libpg-query`, `@pgsql/parser`, etc.) remain +> in the tree for reference and upstream parity, but **this repository does not build +> or publish them**. For the native addon โ€” install, API, platform support, and the +> native-vs-WASM memory comparison โ€” see **[`native/README.md`](native/README.md)**. +> +> ```bash +> npm install @ashbyhq/libpg-query-native +> ``` +> +> The original upstream WASM documentation follows. + # The Real PostgreSQL Parser for JavaScript ### Bring the power of PostgreSQL's native parser to your TypeScript projects โ€” no native builds, no platform headaches. diff --git a/native/.gitignore b/native/.gitignore new file mode 100644 index 0000000..55014c9 --- /dev/null +++ b/native/.gitignore @@ -0,0 +1,7 @@ +.cache/ +prebuilds/ +packages/ +dist/ +node_modules/ +package-lock.json +dist-packs/ diff --git a/native/Makefile b/native/Makefile new file mode 100644 index 0000000..6417aad --- /dev/null +++ b/native/Makefile @@ -0,0 +1,105 @@ +LIBPG_QUERY_REPO := https://github.com/pganalyze/libpg_query.git +LIBPG_QUERY_TAG := 18.0.0 + +OS := $(shell uname -s) +ARCH := $(shell uname -m) + +ifeq ($(OS),Darwin) +PLATFORM := darwin +else ifeq ($(OS),Linux) +PLATFORM := linux +else +$(error Unsupported platform: $(OS)) +endif + +ifeq ($(ARCH),x86_64) +NODE_ARCH := x64 +else ifeq ($(ARCH),aarch64) +NODE_ARCH := arm64 +else ifeq ($(ARCH),arm64) +NODE_ARCH := arm64 +else +NODE_ARCH := $(ARCH) +endif + +# Detect musl libc (Alpine) by the presence of its dynamic loader. +# More reliable than parsing `ldd --version`, which varies across distros. +ifneq ($(wildcard /lib/ld-musl-*),) +PLATFORM_SUFFIX := -musl +else +PLATFORM_SUFFIX := +endif + +# PLATFORM_KEY can be overridden (e.g. `make build PLATFORM_KEY=linux-x64-musl`) +# so CI can pin the output dir to the known target instead of relying on detection. +PLATFORM_KEY ?= $(PLATFORM)-$(NODE_ARCH)$(PLATFORM_SUFFIX) +CACHE_DIR := .cache/$(PLATFORM_KEY) +LIBPG_QUERY_DIR := $(CACHE_DIR)/libpg_query + +LIBPG_QUERY_LIB := $(LIBPG_QUERY_DIR)/libpg_query.a + +NODE_ADDON_API_DIR := $(shell node -p "require('node-addon-api').include_dir" 2>/dev/null || echo "node_modules/node-addon-api") +NODE_INCLUDE := $(shell node -p "require('node-api-headers').include_dir" 2>/dev/null || echo "") + +OUT_DIR := prebuilds/$(PLATFORM_KEY) +OUT_FILE := $(OUT_DIR)/libpg_query_native.node + +CXXFLAGS := -O2 -std=c++17 -fPIC -DNAPI_VERSION=8 -DNODE_ADDON_API_DISABLE_DEPRECATED +LDFLAGS := + +ifeq ($(PLATFORM),darwin) +LDFLAGS += -undefined dynamic_lookup -dynamiclib +else +LDFLAGS += -shared +endif + +INCLUDES := \ + -I$(LIBPG_QUERY_DIR) \ + -I$(LIBPG_QUERY_DIR)/vendor \ + -I$(NODE_ADDON_API_DIR) \ + $(if $(NODE_INCLUDE),-I$(NODE_INCLUDE),) + +# Clone and build libpg_query. Clone into a temp dir and move into place only on +# success, so an interrupted clone can't leave a poisoned (partial) cache dir +# that later builds would mistake for a complete checkout. +$(LIBPG_QUERY_DIR): + mkdir -p $(CACHE_DIR) + rm -rf $(LIBPG_QUERY_DIR).tmp + git clone -b $(LIBPG_QUERY_TAG) --single-branch --depth 1 $(LIBPG_QUERY_REPO) $(LIBPG_QUERY_DIR).tmp + mv $(LIBPG_QUERY_DIR).tmp $(LIBPG_QUERY_DIR) + +$(LIBPG_QUERY_LIB): $(LIBPG_QUERY_DIR) + cd $(LIBPG_QUERY_DIR) && $(MAKE) build + +# Build the .node addon +$(OUT_FILE): src/addon.cc $(LIBPG_QUERY_LIB) + mkdir -p $(OUT_DIR) + $(CXX) $(CXXFLAGS) $(INCLUDES) \ + src/addon.cc \ + $(LIBPG_QUERY_LIB) \ + $(LDFLAGS) \ + -lpthread \ + -o $(OUT_FILE) + @echo "Built $(OUT_FILE) ($$(du -h $(OUT_FILE) | cut -f1))" + +.PHONY: build clean clean-cache deps info + +build: $(OUT_FILE) + +deps: $(LIBPG_QUERY_LIB) + +clean: + rm -rf prebuilds + +clean-cache: + rm -rf .cache + +info: + @echo "Platform: $(PLATFORM_KEY)" + @echo "libpg_query: $(LIBPG_QUERY_TAG)" + @echo "Output: $(OUT_FILE)" + @echo "Node API dir: $(NODE_ADDON_API_DIR)" + @echo "" + @echo "For jemalloc memory benefits, use LD_PRELOAD:" + @echo " Linux: LD_PRELOAD=/usr/lib/x86_64-linux-gnu/libjemalloc.so node app.js" + @echo " macOS: DYLD_INSERT_LIBRARIES=/opt/homebrew/lib/libjemalloc.dylib node app.js" diff --git a/native/README.md b/native/README.md new file mode 100644 index 0000000..220f199 --- /dev/null +++ b/native/README.md @@ -0,0 +1,193 @@ +# @ashbyhq/libpg-query-native + +Native N-API PostgreSQL query parser โ€” a memory-efficient alternative to the WASM build. + +## Why native? + +The WASM build (`@libpg-query/parser`) carries a structural memory cost: WebAssembly +linear memory only ever grows. Once a large parse expands the heap, that memory is +never returned to the OS, so a process that parses one big query keeps the high-water +mark for its lifetime, and repeated large parses ratchet RSS upward monotonically. +No allocator choice can change this โ€” it's a property of the WASM memory model. + +The native build removes that ceiling: it uses the host allocator, so freed memory can +actually be returned to the OS. Pairing it with **jemalloc** (via `LD_PRELOAD` / +`DYLD_INSERT_LIBRARIES`) roughly halves peak RSS and, more importantly, keeps it +*stable* across repeated large parses instead of ratcheting. + +### Measured: native vs WASM + +Parsing a 3.31 MB SQL query (1500ร— `UNION ALL`, ~65 MB JSON parse tree), 3 parse/free +cycles, `darwin-arm64`, Node 24. Each backend measured in its own process. "Retained" +is RSS after the result is dropped and GC settles; throughput is a small query ร—10k. + +| Backend | idle RSS | peak RSS (max of 3) | retained after free | throughput | +|---------|---------|---------------------|---------------------|------------| +| WASM (`@libpg-query/parser`) | 93 MB | 1359 MB | **+1202 MB** (never shrinks) | 125k/s | +| Native โ€” system malloc | 53 MB | 932 MB | +812 MB (ratchets up) | 139k/s | +| Native โ€” **jemalloc** | 55 MB | **498 MB** | **+377 MB** (stabilizes) | 139k/s | + +Per-cycle peak progression: + +``` +WASM: 1261 โ†’ 1359 โ†’ 1359 MB (plateaus at a high permanent floor) +Native system: 649 โ†’ 867 โ†’ 932 MB (fragments, still climbing) +Native jemalloc: 381 โ†’ 497 โ†’ 498 MB (flat after cycle 2) +``` + +WASM linear memory only ever grows, so ~1.2 GB from one big parse is held for the +process lifetime. Native + system malloc is lower but still ratchets. Native + +jemalloc has ~2.7ร— lower peak than WASM, returns freed pages to the OS, and stabilizes. +`MALLOC_CONF=dirty_decay_ms:0,muzzy_decay_ms:0` trims peak a little further (~477 MB). +Throughput is identical across allocators โ€” jemalloc is a pure memory win. + +Reproduce with `node --expose-gc benchmark/memory.mjs --all --cycles 3` (with +`@libpg-query/parser` installed) and `bash benchmark/compare-allocators.sh --cycles 3`. + +## Installation + +```bash +npm install @ashbyhq/libpg-query-native +``` + +Platform-specific binaries ship as separate packages and are installed +automatically via optional dependencies. Each declares `os`/`cpu`/`libc`, so +npm and Yarn install **only** the one matching the host: + +| Package | `os` | `cpu` | `libc` | +|---------|------|-------|--------| +| `@ashbyhq/libpg-query-native-darwin-arm64` | darwin | arm64 | โ€” | +| `@ashbyhq/libpg-query-native-linux-x64` | linux | x64 | glibc | +| `@ashbyhq/libpg-query-native-linux-arm64` | linux | arm64 | glibc | +| `@ashbyhq/libpg-query-native-linux-x64-musl` | linux | x64 | musl | +| `@ashbyhq/libpg-query-native-linux-arm64-musl` | linux | arm64 | musl | + +glibc and musl builds are marked mutually exclusive, so an Alpine host pulls the +musl binary and a Debian/Ubuntu host pulls the glibc one โ€” never both. + +No node-gyp or compiler toolchain needed at install time. + +### Cross-architecture installs + +When building for a target that differs from the install host (e.g. a Linux +Docker image built on an Apple Silicon Mac), tell the package manager which +architectures to fetch: + +```bash +# npm +npm install --os=linux --cpu=x64 --libc=glibc + +# Yarn Berry โ€” in .yarnrc.yml +supportedArchitectures: + os: [linux] + cpu: [x64] + libc: [glibc] +``` + +> Note: Yarn Classic (1.x) honors `os`/`cpu` but not `libc`. On musl hosts it may +> install both Linux variants; the runtime loader still selects the correct one +> via musl detection. + +## Usage + +Drop-in replacement for `@libpg-query/parser`: + +```js +const { parse, parseSync, fingerprint, normalize, scan } = require('@ashbyhq/libpg-query-native'); + +// Sync (no init needed โ€” native loads instantly) +const result = parseSync('SELECT id, name FROM users WHERE active = true'); + +// Async (same result, just wrapped in a promise) +const result2 = await parse('SELECT id, name FROM users WHERE active = true'); +``` + +### API + +| Function | Sync | Async | Returns | +|----------|------|-------|---------| +| `parseSync(sql)` / `parse(sql)` | โœ“ | โœ“ | `ParseResult` (JSON AST) | +| `parsePlPgSQLSync(sql)` / `parsePlPgSQL(sql)` | โœ“ | โœ“ | PL/pgSQL parse tree | +| `fingerprintSync(sql)` / `fingerprint(sql)` | โœ“ | โœ“ | 16-char hex fingerprint | +| `normalizeSync(sql)` / `normalize(sql)` | โœ“ | โœ“ | Normalized query string | +| `scanSync(sql)` / `scan(sql)` | โœ“ | โœ“ | `ScanResult` with tokens | + +## Using jemalloc for optimal memory + +The native addon uses the system allocator by default. For optimal memory behavior +(especially with large queries), preload jemalloc: + +```bash +# Linux +LD_PRELOAD=/usr/lib/x86_64-linux-gnu/libjemalloc.so.2 node app.js + +# macOS (brew install jemalloc) +DYLD_INSERT_LIBRARIES=$(brew --prefix jemalloc)/lib/libjemalloc.dylib node app.js +``` + +Or in your Dockerfile: +```dockerfile +RUN apt-get install -y libjemalloc2 +ENV LD_PRELOAD=/usr/lib/x86_64-linux-gnu/libjemalloc.so.2 +``` + +> The `libjemalloc.so.2` path above is for Debian/Ubuntu on x86_64. It differs by +> distro and architecture (e.g. `/usr/lib/aarch64-linux-gnu/libjemalloc.so.2` on +> arm64, `/usr/lib64/libjemalloc.so.2` on RHEL/Fedora). Find it with +> `ldconfig -p | grep jemalloc`. + +## Benchmarks + +```bash +# Quick sanity check +node --expose-gc benchmark/memory.mjs --small + +# Full benchmark (large query, ~3.31 MB SQL) +node --expose-gc benchmark/memory.mjs + +# Compare with/without jemalloc +bash benchmark/run-with-jemalloc.sh + +# Throughput benchmark +node --expose-gc benchmark/memory.mjs --throughput + +# Compare against WASM (requires @libpg-query/parser installed) +node --expose-gc benchmark/memory.mjs --all + +# CI regression benchmark (emits github-action-benchmark JSON) +node --expose-gc benchmark/ci-bench.mjs --out results.json +``` + +### Regression tracking in CI + +The `Native Benchmark` workflow runs `ci-bench.mjs` on a fixed runner +(ubuntu-24.04, under jemalloc) for every PR and push to `main`. It tracks four +smaller-is-better metrics โ€” large-query parse time, peak RSS, retained RSS, and +small-query latency โ€” against a baseline stored on the `gh-pages` branch. Each +run posts the per-metric difference to the job summary; a regression beyond +**2ร— the baseline** comments on the PR and fails the check. The threshold is +deliberately conservative (`alert-threshold: 200%`) to tolerate shared-runner +noise โ€” tune it in `.github/workflows/native-benchmark.yml`. + +## Building from source + +```bash +cd native +npm install +make build # builds libpg_query + the .node addon +npm run build:ts # compiles TypeScript +npm test +``` + +### Generating platform packages + +After building: +```bash +node scripts/package-platforms.mjs +``` + +This creates `packages/libpg-query-native-/` directories ready for `npm publish`. + +## License + +MIT diff --git a/native/benchmark/breakdown.mjs b/native/benchmark/breakdown.mjs new file mode 100644 index 0000000..e2cd44c --- /dev/null +++ b/native/benchmark/breakdown.mjs @@ -0,0 +1,101 @@ +/** + * Measure where time is spent: native parse vs string copy vs JSON.parse. + * + * Usage: + * node benchmark/breakdown.mjs # large query + * node benchmark/breakdown.mjs --small # small query + */ + +import { argv } from "process"; + +const MB = 1024 * 1024; + +function generateBigQuery(unions = 1500, cols = 400) { + const colList = Array.from({ length: cols }, (_, i) => `c${i}`).join(", "); + const select = `SELECT ${colList} FROM t`; + return Array.from({ length: unions }, () => select).join(" UNION ALL "); +} + +async function main() { + const small = argv.includes("--small"); + const addon = (await import("../dist/index.js")).default || await import("../dist/index.js"); + + // Raw addon for string-level measurement + const { createRequire } = await import("module"); + const require = createRequire(import.meta.url); + const raw = require("../prebuilds/" + process.platform + "-" + process.arch + "/libpg_query_native.node"); + + const query = small + ? "SELECT id, name FROM users WHERE id = $1" + : generateBigQuery(); + + console.log(`Query size: ${(query.length / MB).toFixed(2)} MB`); + console.log(""); + + // Warmup + for (let i = 0; i < 5; i++) raw.parseSync(query); + + const iterations = small ? 10000 : 3; + + // 1. Raw native call (parse + JSON serialize in C + string copy to V8) + let jsonStr; + { + const start = performance.now(); + for (let i = 0; i < iterations; i++) { + const res = raw.parseSync(query); + jsonStr = res.result; + } + const elapsed = performance.now() - start; + const avg = elapsed / iterations; + console.log(`Native (parse + JSON serialize + string copy): ${avg < 1 ? (avg * 1000).toFixed(1) + ' ฮผs' : avg.toFixed(2) + ' ms'}`); + console.log(` JSON string size: ${Buffer.byteLength(jsonStr) < MB ? (Buffer.byteLength(jsonStr) / 1024).toFixed(1) + ' KB' : (Buffer.byteLength(jsonStr) / MB).toFixed(2) + ' MB'}`); + } + + // 2. JSON.parse only + { + const start = performance.now(); + for (let i = 0; i < iterations; i++) { + JSON.parse(jsonStr); + } + const elapsed = performance.now() - start; + const avg = elapsed / iterations; + console.log(`JSON.parse: ${avg < 1 ? (avg * 1000).toFixed(1) + ' ฮผs' : avg.toFixed(2) + ' ms'}`); + } + + // 3. End-to-end (native + JSON.parse) + { + const start = performance.now(); + for (let i = 0; i < iterations; i++) { + const res = raw.parseSync(query); + JSON.parse(res.result); + } + const elapsed = performance.now() - start; + const avgE2e = elapsed / iterations; + console.log(`End-to-end (native + JSON.parse): ${avgE2e < 1 ? (avgE2e * 1000).toFixed(1) + ' ฮผs' : avgE2e.toFixed(2) + ' ms'}`); + } + + // 4. Raw native call returning string only (no JSON.parse) โ€” to isolate + let nativeOnly, jsonParseOnly, e2e; + { + const s1 = performance.now(); + for (let i = 0; i < iterations; i++) raw.parseSync(query); + nativeOnly = (performance.now() - s1) / iterations; + + const s2 = performance.now(); + for (let i = 0; i < iterations; i++) JSON.parse(jsonStr); + jsonParseOnly = (performance.now() - s2) / iterations; + + const s3 = performance.now(); + for (let i = 0; i < iterations; i++) { const r = raw.parseSync(query); JSON.parse(r.result); } + e2e = (performance.now() - s3) / iterations; + } + + const fmt = v => v < 1 ? (v * 1000).toFixed(1) + ' ฮผs' : v.toFixed(2) + ' ms'; + console.log(""); + console.log(`Breakdown (second run, warmed up):`); + console.log(` Native (parse+serialize+copy): ${(nativeOnly / e2e * 100).toFixed(0)}% ${fmt(nativeOnly)}`); + console.log(` JSON.parse: ${(jsonParseOnly / e2e * 100).toFixed(0)}% ${fmt(jsonParseOnly)}`); + console.log(` Total e2e: ${fmt(e2e)}`); +} + +main().catch(console.error); diff --git a/native/benchmark/ci-bench.mjs b/native/benchmark/ci-bench.mjs new file mode 100644 index 0000000..3e4edcb --- /dev/null +++ b/native/benchmark/ci-bench.mjs @@ -0,0 +1,128 @@ +/** + * CI regression benchmark. + * + * Emits results in the github-action-benchmark `customSmallerIsBetter` format + * so the difference vs. the stored baseline is reported on every PR and a + * conservative threshold can fail the build on a real regression. + * + * All metrics are normalized to "smaller is better": + * - large-query parse time (ms) + * - large-query peak RSS (MB) + * - large-query retained RSS after free (MB) + * - small-query latency (ยตs per parse) [inverse of throughput] + * + * Run with: node --expose-gc benchmark/ci-bench.mjs --out results.json + * For meaningful, stable memory numbers, preload jemalloc (see workflow). + */ + +import { argv } from "process"; +import { writeFileSync } from "fs"; +import { createRequire } from "module"; + +const require = createRequire(import.meta.url); +const MB = 1024 * 1024; + +function rss() { + return process.memoryUsage.rss(); +} + +function generateBigQuery(unions = 1500, cols = 400) { + const colList = Array.from({ length: cols }, (_, i) => `c${i}`).join(", "); + const select = `SELECT ${colList} FROM t`; + return Array.from({ length: unions }, () => select).join(" UNION ALL "); +} + +async function gcSettle(ms = 300) { + if (global.gc) global.gc(); + await new Promise((r) => setTimeout(r, ms)); + if (global.gc) global.gc(); + await new Promise((r) => setTimeout(r, ms)); +} + +function detectAllocator() { + const preload = + process.env.LD_PRELOAD || process.env.DYLD_INSERT_LIBRARIES || ""; + if (preload.includes("jemalloc")) return "jemalloc"; + if (preload.includes("mimalloc")) return "mimalloc"; + if (preload.includes("tcmalloc")) return "tcmalloc"; + return "system"; +} + +async function main() { + const outIdx = argv.indexOf("--out"); + const outFile = outIdx !== -1 ? argv[outIdx + 1] : "benchmark-results.json"; + + const lib = require("../dist/index.js"); + const allocator = detectAllocator(); + + const bigQuery = generateBigQuery(); + const smallQuery = "SELECT id, name FROM users WHERE id = $1 AND active = true"; + + // Warm up + for (let i = 0; i < 5; i++) lib.parseSync(bigQuery.slice(0, 200)); + + // --- Large query: parse time, peak RSS, retained RSS (3 cycles) --- + await gcSettle(); + const idleRss = rss(); + let peakRss = 0; + let totalParse = 0; + const cycles = 3; + for (let c = 0; c < cycles; c++) { + const start = performance.now(); + let result = lib.parseSync(bigQuery); + totalParse += performance.now() - start; + const cur = rss(); + if (cur > peakRss) peakRss = cur; + result = null; + await gcSettle(); + } + await gcSettle(); + const retainedRss = rss() - idleRss; + const avgParseMs = totalParse / cycles; + + // --- Small query: throughput -> ยตs/parse --- + const iterations = 20000; + for (let i = 0; i < 1000; i++) lib.parseSync(smallQuery); // warm + const tStart = performance.now(); + for (let i = 0; i < iterations; i++) lib.parseSync(smallQuery); + const elapsedMs = performance.now() - tStart; + const usPerParse = (elapsedMs * 1000) / iterations; + + const results = [ + { + name: "Large query parse time", + unit: "ms", + value: round(avgParseMs), + }, + { + name: "Large query peak RSS", + unit: "MB", + value: round(peakRss / MB), + }, + { + name: "Large query retained RSS", + unit: "MB", + value: round(retainedRss / MB), + }, + { + name: "Small query latency", + unit: "us/parse", + value: round(usPerParse, 3), + }, + ]; + + console.log(`Allocator: ${allocator}`); + console.table(results); + writeFileSync(outFile, JSON.stringify(results, null, 2) + "\n"); + console.log(`Wrote ${outFile}`); +} + +function round(n, dp = 1) { + const f = 10 ** dp; + return Math.round(n * f) / f; +} + +main().catch((e) => { + console.error(e); + process.exit(1); +}); diff --git a/native/benchmark/compare-allocators.sh b/native/benchmark/compare-allocators.sh new file mode 100755 index 0000000..c96dc58 --- /dev/null +++ b/native/benchmark/compare-allocators.sh @@ -0,0 +1,94 @@ +#!/bin/bash +# Compare allocator behavior for the native libpg-query addon. +# +# Tests: system malloc, jemalloc, mimalloc, tcmalloc +# Measures: idle RSS, peak RSS, post-free RSS, throughput +# +# Usage: +# bash benchmark/compare-allocators.sh # large query (default) +# bash benchmark/compare-allocators.sh --small # small query sanity check +# bash benchmark/compare-allocators.sh --throughput # include throughput numbers + +set -e + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +BENCHMARK="$SCRIPT_DIR/memory.mjs" +ARGS=("$@") + +# Detect allocator libraries +JEMALLOC_LIB="" +MIMALLOC_LIB="" +TCMALLOC_LIB="" + +if [[ "$(uname)" == "Darwin" ]]; then + JEMALLOC_LIB="$(brew --prefix jemalloc 2>/dev/null)/lib/libjemalloc.dylib" + MIMALLOC_LIB="$(brew --prefix mimalloc 2>/dev/null)/lib/libmimalloc.dylib" + TCMALLOC_LIB="$(brew --prefix gperftools 2>/dev/null)/lib/libtcmalloc.dylib" + PRELOAD_VAR="DYLD_INSERT_LIBRARIES" +else + for candidate in \ + /usr/lib/x86_64-linux-gnu/libjemalloc.so.2 \ + /usr/lib/aarch64-linux-gnu/libjemalloc.so.2 \ + /usr/lib64/libjemalloc.so.2 \ + /usr/lib/libjemalloc.so; do + [[ -f "$candidate" ]] && JEMALLOC_LIB="$candidate" && break + done + for candidate in \ + /usr/lib/x86_64-linux-gnu/libmimalloc.so \ + /usr/lib/aarch64-linux-gnu/libmimalloc.so \ + /usr/lib64/libmimalloc.so \ + /usr/lib/libmimalloc.so; do + [[ -f "$candidate" ]] && MIMALLOC_LIB="$candidate" && break + done + for candidate in \ + /usr/lib/x86_64-linux-gnu/libtcmalloc.so \ + /usr/lib/aarch64-linux-gnu/libtcmalloc.so \ + /usr/lib64/libtcmalloc.so \ + /usr/lib/libtcmalloc.so; do + [[ -f "$candidate" ]] && TCMALLOC_LIB="$candidate" && break + done + PRELOAD_VAR="LD_PRELOAD" +fi + +echo "=============================================" +echo " Allocator Comparison Benchmark" +echo " $(uname -s) $(uname -m), Node $(node -v)" +echo "=============================================" +echo "" +echo "Allocators found:" +echo " system malloc: yes (always available)" +[[ -f "$JEMALLOC_LIB" ]] && echo " jemalloc: $JEMALLOC_LIB" || echo " jemalloc: not found" +[[ -f "$MIMALLOC_LIB" ]] && echo " mimalloc: $MIMALLOC_LIB" || echo " mimalloc: not found" +[[ -f "$TCMALLOC_LIB" ]] && echo " tcmalloc: $TCMALLOC_LIB" || echo " tcmalloc: not found" +echo "" + +run_benchmark() { + local name="$1" + local preload="$2" + + echo ">>> $name" + if [[ -n "$preload" ]]; then + env "$PRELOAD_VAR=$preload" node --expose-gc "$BENCHMARK" "${ARGS[@]}" 2>&1 + else + node --expose-gc "$BENCHMARK" "${ARGS[@]}" 2>&1 + fi + echo "" +} + +run_benchmark "System malloc (default)" "" + +[[ -f "$JEMALLOC_LIB" ]] && run_benchmark "jemalloc" "$JEMALLOC_LIB" +[[ -f "$MIMALLOC_LIB" ]] && run_benchmark "mimalloc" "$MIMALLOC_LIB" +[[ -f "$TCMALLOC_LIB" ]] && run_benchmark "tcmalloc" "$TCMALLOC_LIB" + +# jemalloc with aggressive decay +if [[ -f "$JEMALLOC_LIB" ]]; then + echo ">>> jemalloc (aggressive decay: dirty=0, muzzy=0)" + env "$PRELOAD_VAR=$JEMALLOC_LIB" MALLOC_CONF="dirty_decay_ms:0,muzzy_decay_ms:0" \ + node --expose-gc "$BENCHMARK" "${ARGS[@]}" 2>&1 + echo "" +fi + +echo "=============================================" +echo " Done. Compare Peak RSS and Retained columns." +echo "=============================================" diff --git a/native/benchmark/memory.mjs b/native/benchmark/memory.mjs new file mode 100644 index 0000000..773ccb6 --- /dev/null +++ b/native/benchmark/memory.mjs @@ -0,0 +1,243 @@ +/** + * Memory benchmark: compare allocator behavior for large SQL parses. + * + * Generates a ~3.3 MB SQL query (1500ร— UNION ALL of SELECT c0..c399 FROM t), + * parses it, and measures RSS at idle, peak (post-parse), and after free. + * + * Usage: + * node benchmark/memory.mjs # native backend + * node benchmark/memory.mjs --wasm # WASM backend for comparison + * node benchmark/memory.mjs --all # run all available backends + * node benchmark/memory.mjs --small # quick sanity check (small query) + * node benchmark/memory.mjs --throughput # throughput benchmark + * node benchmark/memory.mjs --cycles 3 # multiple parse/free cycles + */ + +import { argv } from "process"; + +const MB = 1024 * 1024; + +function rss() { + return process.memoryUsage.rss(); +} + +function generateBigQuery(unions = 1500, cols = 400) { + const colList = Array.from({ length: cols }, (_, i) => `c${i}`).join(", "); + const select = `SELECT ${colList} FROM t`; + return Array.from({ length: unions }, () => select).join(" UNION ALL "); +} + +function generateSmallQuery() { + return "SELECT id, name, email, created_at FROM users WHERE id = $1 AND active = true"; +} + +function detectAllocator() { + const preload = process.env.LD_PRELOAD || process.env.DYLD_INSERT_LIBRARIES || ""; + if (preload.includes("jemalloc")) return "jemalloc"; + if (preload.includes("mimalloc")) return "mimalloc"; + if (preload.includes("tcmalloc")) return "tcmalloc"; + return "system"; +} + +async function gcAndSettle(ms = 200) { + if (global.gc) global.gc(); + await new Promise((r) => setTimeout(r, ms)); + if (global.gc) global.gc(); + await new Promise((r) => setTimeout(r, ms)); +} + +async function benchmarkParse(backend, label, query, cycles = 1) { + await gcAndSettle(); + + const idleRss = rss(); + console.log(`\n--- ${label} ---`); + console.log(`Query size: ${(query.length / MB).toFixed(2)} MB`); + console.log(`Idle RSS: ${(idleRss / MB).toFixed(1)} MB`); + + let peakRss = 0; + let totalParseTime = 0; + let resultSize = 0; + + for (let cycle = 0; cycle < cycles; cycle++) { + if (cycles > 1) process.stdout.write(` cycle ${cycle + 1}/${cycles}...`); + + const start = performance.now(); + let result; + try { + result = await backend.parse(query); + } catch (e) { + console.log(` parse failed: ${e.message}`); + return null; + } + const elapsed = performance.now() - start; + totalParseTime += elapsed; + + const currentRss = rss(); + if (currentRss > peakRss) peakRss = currentRss; + + if (cycle === 0) { + const resultJson = JSON.stringify(result); + resultSize = Buffer.byteLength(resultJson); + } + + // Drop references and let GC + allocator return memory + result = null; + await gcAndSettle(300); + + const afterCycleRss = rss(); + if (cycles > 1) { + console.log( + ` peak=${(currentRss / MB).toFixed(0)} MB, after=${(afterCycleRss / MB).toFixed(0)} MB, ${elapsed.toFixed(0)} ms` + ); + } + } + + const afterRss = rss(); + const avgParseTime = totalParseTime / cycles; + + console.log(`Parse time: ${avgParseTime.toFixed(0)} ms${cycles > 1 ? ` (avg of ${cycles})` : ""}`); + console.log(`Result size: ${(resultSize / MB).toFixed(1)} MB`); + console.log(`Peak RSS: ${(peakRss / MB).toFixed(1)} MB`); + console.log(`Peak delta: +${((peakRss - idleRss) / MB).toFixed(1)} MB`); + console.log(`After free: ${(afterRss / MB).toFixed(1)} MB`); + console.log(`Retained: +${((afterRss - idleRss) / MB).toFixed(1)} MB`); + + return { + label, + idleRss, + peakRss, + afterRss, + parseTime: avgParseTime, + resultSize, + querySize: query.length, + }; +} + +async function benchmarkThroughput(backend, label, iterations = 10000) { + const query = "SELECT id, name FROM users WHERE id = $1"; + + // Warmup + for (let i = 0; i < 100; i++) { + backend.parseSync(query); + } + + const start = performance.now(); + for (let i = 0; i < iterations; i++) { + backend.parseSync(query); + } + const elapsed = performance.now() - start; + const opsPerSec = Math.round((iterations / elapsed) * 1000); + + console.log(`\n--- ${label} throughput ---`); + console.log(`${iterations} parses in ${elapsed.toFixed(0)} ms`); + console.log(`${opsPerSec.toLocaleString()} ops/sec`); + + return { label, iterations, elapsed, opsPerSec }; +} + +async function loadNative() { + try { + return await import("../dist/index.js"); + } catch { + console.error( + "Native backend not found. Run 'make build && npm run build:ts' first." + ); + return null; + } +} + +async function loadWasm() { + try { + const wasm = await import("@libpg-query/parser"); + if (wasm.loadModule) await wasm.loadModule(); + return wasm; + } catch { + console.error( + "WASM backend not found. Install @libpg-query/parser for comparison." + ); + return null; + } +} + +async function main() { + const flags = new Set(argv.slice(2)); + + const runAll = flags.has("--all"); + const runWasm = flags.has("--wasm") || runAll; + const runNative = !flags.has("--wasm") || runAll; + const small = flags.has("--small"); + const throughput = flags.has("--throughput"); + + let cycles = 1; + const cyclesIdx = argv.indexOf("--cycles"); + if (cyclesIdx !== -1 && argv[cyclesIdx + 1]) { + cycles = parseInt(argv[cyclesIdx + 1], 10) || 1; + } + + const query = small ? generateSmallQuery() : generateBigQuery(); + const allocator = detectAllocator(); + + console.log("=== libpg-query Memory Benchmark ==="); + console.log(`Node ${process.version}, ${process.platform}-${process.arch}`); + console.log(`Allocator: ${allocator}`); + console.log(`GC exposed: ${typeof global.gc === "function"}`); + if (typeof global.gc !== "function") { + console.log("Hint: run with --expose-gc for more accurate measurements"); + } + + const results = []; + + if (runNative) { + const native = await loadNative(); + if (native) { + const label = `Native (${allocator})`; + results.push(await benchmarkParse(native, label, query, cycles)); + if (throughput) { + await benchmarkThroughput(native, label); + } + } else { + // A requested backend failed to load โ€” fail loudly rather than + // reporting a "successful" run that benchmarked nothing. + process.exitCode = 1; + } + } + + if (runWasm) { + const wasm = await loadWasm(); + if (wasm) { + results.push(await benchmarkParse(wasm, "WASM (emscripten)", query, cycles)); + if (throughput) { + await benchmarkThroughput(wasm, "WASM (emscripten)"); + } + } else { + process.exitCode = 1; + } + } + + // Summary table + const valid = results.filter(Boolean); + if (valid.length > 1) { + console.log("\n=== Summary ==="); + console.log( + "Backend".padEnd(25) + + "Idle".padStart(10) + + "Peak".padStart(10) + + "After".padStart(10) + + "Retained".padStart(10) + + "Parse ms".padStart(10) + ); + console.log("-".repeat(75)); + for (const r of valid) { + console.log( + r.label.padEnd(25) + + `${(r.idleRss / MB).toFixed(1)} MB`.padStart(10) + + `${(r.peakRss / MB).toFixed(1)} MB`.padStart(10) + + `${(r.afterRss / MB).toFixed(1)} MB`.padStart(10) + + `+${((r.afterRss - r.idleRss) / MB).toFixed(1)} MB`.padStart(10) + + `${r.parseTime.toFixed(0)}`.padStart(10) + ); + } + } +} + +main().catch(console.error); diff --git a/native/benchmark/run-with-jemalloc.sh b/native/benchmark/run-with-jemalloc.sh new file mode 100755 index 0000000..d6174b4 --- /dev/null +++ b/native/benchmark/run-with-jemalloc.sh @@ -0,0 +1,57 @@ +#!/bin/bash +# Run the memory benchmark with jemalloc preloaded. +# +# Prerequisites: +# macOS: brew install jemalloc +# Linux: apt install libjemalloc-dev OR yum install jemalloc-devel + +set -e + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" + +# Find jemalloc +if [[ "$(uname)" == "Darwin" ]]; then + JEMALLOC_LIB="${JEMALLOC_LIB:-$(brew --prefix jemalloc 2>/dev/null)/lib/libjemalloc.dylib}" + if [[ ! -f "$JEMALLOC_LIB" ]]; then + echo "jemalloc not found. Install with: brew install jemalloc" + echo "Or set JEMALLOC_LIB=/path/to/libjemalloc.dylib" + exit 1 + fi + echo "Using jemalloc: $JEMALLOC_LIB" + echo "" + + echo "=== Without jemalloc ===" + node --expose-gc "$SCRIPT_DIR/memory.mjs" "$@" + echo "" + echo "=== With jemalloc (DYLD_INSERT_LIBRARIES) ===" + DYLD_INSERT_LIBRARIES="$JEMALLOC_LIB" node --expose-gc "$SCRIPT_DIR/memory.mjs" "$@" +else + JEMALLOC_LIB="${JEMALLOC_LIB:-}" + if [[ -z "$JEMALLOC_LIB" ]]; then + for candidate in \ + /usr/lib/x86_64-linux-gnu/libjemalloc.so.2 \ + /usr/lib/aarch64-linux-gnu/libjemalloc.so.2 \ + /usr/lib64/libjemalloc.so.2 \ + /usr/lib/libjemalloc.so.2 \ + /usr/lib/x86_64-linux-gnu/libjemalloc.so \ + /usr/lib/libjemalloc.so; do + if [[ -f "$candidate" ]]; then + JEMALLOC_LIB="$candidate" + break + fi + done + fi + if [[ -z "$JEMALLOC_LIB" || ! -f "$JEMALLOC_LIB" ]]; then + echo "jemalloc not found. Install with: apt install libjemalloc-dev" + echo "Or set JEMALLOC_LIB=/path/to/libjemalloc.so" + exit 1 + fi + echo "Using jemalloc: $JEMALLOC_LIB" + echo "" + + echo "=== Without jemalloc ===" + node --expose-gc "$SCRIPT_DIR/memory.mjs" "$@" + echo "" + echo "=== With jemalloc (LD_PRELOAD) ===" + LD_PRELOAD="$JEMALLOC_LIB" node --expose-gc "$SCRIPT_DIR/memory.mjs" "$@" +fi diff --git a/native/package.json b/native/package.json new file mode 100644 index 0000000..5f1bbc2 --- /dev/null +++ b/native/package.json @@ -0,0 +1,60 @@ +{ + "name": "@ashbyhq/libpg-query-native", + "version": "0.1.1-beta.0", + "description": "PostgreSQL query parser โ€” native N-API + jemalloc backend", + "main": "./dist/index.js", + "types": "./dist/index.d.ts", + "exports": { + ".": { + "import": "./dist/index.js", + "require": "./dist/index.js", + "types": "./dist/index.d.ts" + } + }, + "files": [ + "dist/**/*", + "platforms.json" + ], + "publishConfig": { + "access": "public" + }, + "scripts": { + "build:native": "make build", + "build:ts": "tsc", + "build": "npm run build:native && npm run build:ts", + "clean": "make clean && rm -rf dist", + "test": "node --test test/parsing.test.js test/fingerprint.test.js test/normalize.test.js test/plpgsql.test.js test/scan.test.js test/errors.test.js", + "benchmark": "node benchmark/memory.mjs", + "package-platforms": "node scripts/package-platforms.mjs" + }, + "keywords": [ + "postgresql", + "parser", + "sql", + "native", + "napi", + "jemalloc" + ], + "license": "MIT", + "repository": { + "type": "git", + "url": "git://github.com/ashbyhq/libpg-query-node.git", + "directory": "native" + }, + "dependencies": { + "@pgsql/types": "^18.0.0" + }, + "devDependencies": { + "@types/node": "^26.0.1", + "node-addon-api": "^8.3.1", + "node-api-headers": "^1.5.0", + "typescript": "5.9.3" + }, + "optionalDependencies": { + "@ashbyhq/libpg-query-native-darwin-arm64": "0.1.1-beta.0", + "@ashbyhq/libpg-query-native-linux-x64": "0.1.1-beta.0", + "@ashbyhq/libpg-query-native-linux-arm64": "0.1.1-beta.0", + "@ashbyhq/libpg-query-native-linux-x64-musl": "0.1.1-beta.0", + "@ashbyhq/libpg-query-native-linux-arm64-musl": "0.1.1-beta.0" + } +} diff --git a/native/platforms.json b/native/platforms.json new file mode 100644 index 0000000..994b659 --- /dev/null +++ b/native/platforms.json @@ -0,0 +1,33 @@ +{ + "darwin-arm64": { + "os": ["darwin"], + "cpu": ["arm64"], + "runner": "macos-14" + }, + "linux-x64": { + "os": ["linux"], + "cpu": ["x64"], + "libc": ["glibc"], + "runner": "ubuntu-24.04" + }, + "linux-arm64": { + "os": ["linux"], + "cpu": ["arm64"], + "libc": ["glibc"], + "runner": "ubuntu-24.04-arm" + }, + "linux-x64-musl": { + "os": ["linux"], + "cpu": ["x64"], + "libc": ["musl"], + "runner": "ubuntu-24.04", + "container": "node:24-alpine" + }, + "linux-arm64-musl": { + "os": ["linux"], + "cpu": ["arm64"], + "libc": ["musl"], + "runner": "ubuntu-24.04-arm", + "container": "node:24-alpine" + } +} diff --git a/native/scripts/package-platforms.mjs b/native/scripts/package-platforms.mjs new file mode 100644 index 0000000..37a86f8 --- /dev/null +++ b/native/scripts/package-platforms.mjs @@ -0,0 +1,80 @@ +/** + * Generate per-platform npm packages from prebuilds. + * Each package contains just the .node binary and a package.json. + * + * Run after `make build` to create packages/ directory: + * node scripts/package-platforms.mjs + */ + +import { readdirSync, mkdirSync, copyFileSync, writeFileSync, existsSync, readFileSync } from "fs"; +import { join, basename } from "path"; + +const pkg = JSON.parse(readFileSync("package.json", "utf8")); +const VERSION = pkg.version; +const SCOPE = "@ashbyhq"; +const BASE_NAME = "libpg-query-native"; +const PREBUILDS_DIR = "prebuilds"; +const OUT_DIR = "packages"; + +const PLATFORMS = JSON.parse(readFileSync("platforms.json", "utf8")); + +if (!existsSync(PREBUILDS_DIR)) { + console.error(`No ${PREBUILDS_DIR}/ directory found. Run 'make build' first.`); + process.exit(1); +} + +const platforms = readdirSync(PREBUILDS_DIR); + +for (const platform of platforms) { + const meta = PLATFORMS[platform]; + if (!meta) { + console.warn(`Skipping unknown platform: ${platform}`); + continue; + } + + const pkgName = `${SCOPE}/${BASE_NAME}-${platform}`; + const pkgDir = join(OUT_DIR, `${BASE_NAME}-${platform}`); + + mkdirSync(pkgDir, { recursive: true }); + + // Copy the .node file + const nodeFile = join(PREBUILDS_DIR, platform, "libpg_query_native.node"); + if (!existsSync(nodeFile)) { + console.warn(`No .node file for ${platform}, skipping`); + continue; + } + copyFileSync(nodeFile, join(pkgDir, "libpg_query_native.node")); + + // Write index.js that just re-exports the binary + writeFileSync( + join(pkgDir, "index.js"), + `module.exports = require('./libpg_query_native.node');\n` + ); + + // Write package.json + const pkg = { + name: pkgName, + version: VERSION, + description: `libpg-query native addon for ${platform}`, + main: "index.js", + files: ["index.js", "libpg_query_native.node"], + os: meta.os, + cpu: meta.cpu, + ...(meta.libc ? { libc: meta.libc } : {}), + license: "MIT", + repository: { + type: "git", + url: "git://github.com/ashbyhq/libpg-query-node.git", + directory: `native/packages/${BASE_NAME}-${platform}`, + }, + publishConfig: { + access: "public", + }, + }; + + writeFileSync(join(pkgDir, "package.json"), JSON.stringify(pkg, null, 2) + "\n"); + + console.log(`Created ${pkgName} (${nodeFile})`); +} + +console.log(`\nDone. Platform packages in ${OUT_DIR}/`); diff --git a/native/scripts/sync-optional-deps.mjs b/native/scripts/sync-optional-deps.mjs new file mode 100644 index 0000000..f9b18d6 --- /dev/null +++ b/native/scripts/sync-optional-deps.mjs @@ -0,0 +1,21 @@ +/** + * Sync optionalDependencies in package.json from platforms.json. + * Run after adding/removing platforms: + * node scripts/sync-optional-deps.mjs + */ + +import { readFileSync, writeFileSync } from "fs"; + +const platforms = JSON.parse(readFileSync("platforms.json", "utf8")); +const pkg = JSON.parse(readFileSync("package.json", "utf8")); + +const optionalDeps = {}; +for (const platform of Object.keys(platforms)) { + optionalDeps[`@ashbyhq/libpg-query-native-${platform}`] = pkg.version; +} + +pkg.optionalDependencies = optionalDeps; + +writeFileSync("package.json", JSON.stringify(pkg, null, 2) + "\n"); + +console.log(`Synced ${Object.keys(optionalDeps).length} optionalDependencies from platforms.json`); diff --git a/native/src/addon.cc b/native/src/addon.cc new file mode 100644 index 0000000..0914f84 --- /dev/null +++ b/native/src/addon.cc @@ -0,0 +1,247 @@ +#include +#include +#include + +extern "C" { +#include "pg_query.h" +#include "protobuf/pg_query.pb-c.h" +} + +static std::string EscapeJsonString(const std::string &s) { + std::string out; + out.reserve(s.size() + 8); + for (char c : s) { + switch (c) { + case '"': out += "\\\""; break; + case '\\': out += "\\\\"; break; + case '\n': out += "\\n"; break; + case '\r': out += "\\r"; break; + case '\t': out += "\\t"; break; + case '\b': out += "\\b"; break; + case '\f': out += "\\f"; break; + default: + // Escape any remaining control characters (<0x20) as \u00XX so the + // payload stays valid JSON for JSON.parse() on the JS side. + if (static_cast(c) < 0x20) { + char buf[7]; + std::snprintf(buf, sizeof(buf), "\\u%04x", static_cast(c)); + out += buf; + } else { + out += c; + } + } + } + return out; +} + +// Build a JSON object with error details from a PgQueryError. +static std::string BuildErrorJson(PgQueryError *error) { + std::string msg = error->message ? error->message : "Unknown error"; + std::string json = "{\"message\":\"" + EscapeJsonString(msg) + "\""; + json += ",\"cursorPosition\":" + std::to_string(error->cursorpos > 0 ? error->cursorpos - 1 : 0); + if (error->funcname) + json += ",\"functionName\":\"" + EscapeJsonString(error->funcname) + "\""; + if (error->filename) + json += ",\"fileName\":\"" + EscapeJsonString(error->filename) + "\""; + if (error->lineno > 0) + json += ",\"lineNumber\":" + std::to_string(error->lineno); + if (error->context) + json += ",\"context\":\"" + EscapeJsonString(error->context) + "\""; + json += "}"; + return json; +} + +// Return a {error: json, result: null} object to JS. JS throws from there. +static Napi::Value ReturnError(Napi::Env env, PgQueryError *error) { + Napi::Object obj = Napi::Object::New(env); + obj.Set("error", Napi::String::New(env, BuildErrorJson(error))); + obj.Set("result", env.Null()); + return obj; +} + +static Napi::Value ReturnResult(Napi::Env env, const std::string &result) { + Napi::Object obj = Napi::Object::New(env); + obj.Set("error", env.Null()); + obj.Set("result", Napi::String::New(env, result)); + return obj; +} + +static std::string ValidateQuery(Napi::Env env, const Napi::CallbackInfo &info) { + if (info.Length() < 1 || !info[0].IsString()) { + Napi::TypeError::New(env, "Expected a string argument").ThrowAsJavaScriptException(); + return ""; + } + std::string query = info[0].As().Utf8Value(); + if (query.empty()) { + Napi::Error::New(env, "Query cannot be empty").ThrowAsJavaScriptException(); + return ""; + } + return query; +} + +static Napi::Value ParseSync(const Napi::CallbackInfo &info) { + Napi::Env env = info.Env(); + std::string query = ValidateQuery(env, info); + if (env.IsExceptionPending()) return env.Undefined(); + + PgQueryParseResult result = pg_query_parse(query.c_str()); + + if (result.error) { + Napi::Value ret = ReturnError(env, result.error); + pg_query_free_parse_result(result); + return ret; + } + + std::string json(result.parse_tree); + pg_query_free_parse_result(result); + return ReturnResult(env, json); +} + +static Napi::Value ParsePlPgSQLSync(const Napi::CallbackInfo &info) { + Napi::Env env = info.Env(); + std::string query = ValidateQuery(env, info); + if (env.IsExceptionPending()) return env.Undefined(); + + PgQueryPlpgsqlParseResult result = pg_query_parse_plpgsql(query.c_str()); + + if (result.error) { + Napi::Value ret = ReturnError(env, result.error); + pg_query_free_plpgsql_parse_result(result); + return ret; + } + + std::string json = std::string("{\"plpgsql_funcs\":") + + (result.plpgsql_funcs ? result.plpgsql_funcs : "[]") + "}"; + pg_query_free_plpgsql_parse_result(result); + return ReturnResult(env, json); +} + +static Napi::Value FingerprintSync(const Napi::CallbackInfo &info) { + Napi::Env env = info.Env(); + std::string query = ValidateQuery(env, info); + if (env.IsExceptionPending()) return env.Undefined(); + + PgQueryFingerprintResult result = pg_query_fingerprint(query.c_str()); + + if (result.error) { + Napi::Value ret = ReturnError(env, result.error); + pg_query_free_fingerprint_result(result); + return ret; + } + + std::string fp(result.fingerprint_str); + pg_query_free_fingerprint_result(result); + return ReturnResult(env, fp); +} + +static Napi::Value NormalizeSync(const Napi::CallbackInfo &info) { + Napi::Env env = info.Env(); + std::string query = ValidateQuery(env, info); + if (env.IsExceptionPending()) return env.Undefined(); + + PgQueryNormalizeResult result = pg_query_normalize(query.c_str()); + + if (result.error) { + Napi::Value ret = ReturnError(env, result.error); + pg_query_free_normalize_result(result); + return ret; + } + + std::string normalized(result.normalized_query); + pg_query_free_normalize_result(result); + return ReturnResult(env, normalized); +} + +static std::string GetTokenName(int token_type) { + switch (token_type) { + case 258: return "IDENT"; + case 261: return "SCONST"; + case 266: return "ICONST"; + case 260: return "FCONST"; + case 267: return "PARAM"; + case 40: return "ASCII_40"; + case 41: return "ASCII_41"; + case 42: return "ASCII_42"; + case 44: return "ASCII_44"; + case 59: return "ASCII_59"; + case 61: return "ASCII_61"; + case 268: return "TYPECAST"; + case 272: return "LESS_EQUALS"; + case 273: return "GREATER_EQUALS"; + case 274: return "NOT_EQUALS"; + case 275: return "SQL_COMMENT"; + case 276: return "C_COMMENT"; + default: return "UNKNOWN"; + } +} + +static std::string GetKeywordName(int kind) { + switch (kind) { + case 0: return "NO_KEYWORD"; + case 1: return "UNRESERVED_KEYWORD"; + case 2: return "COL_NAME_KEYWORD"; + case 3: return "TYPE_FUNC_NAME_KEYWORD"; + case 4: return "RESERVED_KEYWORD"; + default: return "UNKNOWN_KEYWORD"; + } +} + +static Napi::Value ScanSync(const Napi::CallbackInfo &info) { + Napi::Env env = info.Env(); + std::string query = ValidateQuery(env, info); + if (env.IsExceptionPending()) return env.Undefined(); + + PgQueryScanResult result = pg_query_scan(query.c_str()); + + if (result.error) { + Napi::Value ret = ReturnError(env, result.error); + pg_query_free_scan_result(result); + return ret; + } + + PgQuery__ScanResult *scan_result = pg_query__scan_result__unpack( + NULL, result.pbuf.len, (const uint8_t *)result.pbuf.data); + + if (!scan_result) { + pg_query_free_scan_result(result); + Napi::Error::New(env, "Failed to unpack scan result").ThrowAsJavaScriptException(); + return env.Undefined(); + } + + Napi::Object scan_obj = Napi::Object::New(env); + scan_obj.Set("version", Napi::Number::New(env, scan_result->version)); + + Napi::Array tokens = Napi::Array::New(env, scan_result->n_tokens); + for (size_t i = 0; i < scan_result->n_tokens; i++) { + PgQuery__ScanToken *token = scan_result->tokens[i]; + Napi::Object tok = Napi::Object::New(env); + tok.Set("start", Napi::Number::New(env, token->start)); + tok.Set("end", Napi::Number::New(env, token->end)); + tok.Set("text", Napi::String::New(env, query.substr(token->start, token->end - token->start))); + tok.Set("tokenType", Napi::Number::New(env, token->token)); + tok.Set("tokenName", Napi::String::New(env, GetTokenName(token->token))); + tok.Set("keywordKind", Napi::Number::New(env, token->keyword_kind)); + tok.Set("keywordName", Napi::String::New(env, GetKeywordName(token->keyword_kind))); + tokens[i] = tok; + } + scan_obj.Set("tokens", tokens); + + pg_query__scan_result__free_unpacked(scan_result, NULL); + pg_query_free_scan_result(result); + + Napi::Object obj = Napi::Object::New(env); + obj.Set("error", env.Null()); + obj.Set("result", scan_obj); + return obj; +} + +Napi::Object Init(Napi::Env env, Napi::Object exports) { + exports.Set("parseSync", Napi::Function::New(env, ParseSync)); + exports.Set("parsePlPgSQLSync", Napi::Function::New(env, ParsePlPgSQLSync)); + exports.Set("fingerprintSync", Napi::Function::New(env, FingerprintSync)); + exports.Set("normalizeSync", Napi::Function::New(env, NormalizeSync)); + exports.Set("scanSync", Napi::Function::New(env, ScanSync)); + return exports; +} + +NODE_API_MODULE(libpg_query_native, Init) diff --git a/native/src/index.ts b/native/src/index.ts new file mode 100644 index 0000000..f7994a7 --- /dev/null +++ b/native/src/index.ts @@ -0,0 +1,184 @@ +import type { ParseResult } from "@pgsql/types"; + +export type { ParseResult } from "@pgsql/types"; + +export interface ScanToken { + start: number; + end: number; + text: string; + tokenType: number; + tokenName: string; + keywordKind: number; + keywordName: string; +} + +export interface ScanResult { + version: number; + tokens: ScanToken[]; +} + +export interface SqlErrorDetails { + message: string; + cursorPosition: number; + fileName?: string; + functionName?: string; + lineNumber?: number; + context?: string; +} + +export class SqlError extends Error { + sqlDetails?: SqlErrorDetails; + + constructor(message: string, details?: SqlErrorDetails) { + super(message); + this.name = "SqlError"; + this.sqlDetails = details; + } +} + +export function hasSqlDetails(error: unknown): error is SqlError { + return error instanceof SqlError && error.sqlDetails !== undefined; +} + +function loadNativeAddon(): NativeAddon { + const platform = process.platform; + const arch = process.arch; + const musl = isMusl(); + + const platformKey = musl ? `${platform}-${arch}-musl` : `${platform}-${arch}`; + + // Try loading from a local prebuild first (development / monorepo) + const path = require("path"); + const localPrebuild = path.join( + __dirname, + "..", + "prebuilds", + platformKey, + "libpg_query_native.node" + ); + try { + return require(localPrebuild); + } catch { + // fall through to platform package + } + + const platforms: Record = require("../platforms.json"); + const pkgName = `@ashbyhq/libpg-query-native-${platformKey}`; + + if (!platforms[platformKey]) { + throw new Error( + `Unsupported platform: ${platformKey}. ` + + `Supported: ${Object.keys(platforms).join(", ")}` + ); + } + + try { + return require(pkgName); + } catch { + throw new Error( + `Native addon not found for ${platformKey}. Install ${pkgName} or ensure it's in your dependencies.` + ); + } +} + +function isMusl(): boolean { + if (process.platform !== "linux") return false; + // Prefer Node's own libc marker: `glibcVersionRuntime` is set on glibc and + // absent on musl. This is reliable in minimal images (distroless/scratch) + // that ship no `ldd` binary โ€” shelling out there throws and would otherwise + // be misread as musl, selecting the wrong platform package. + try { + const report = process.report?.getReport?.() as + | { header?: { glibcVersionRuntime?: string } } + | undefined; + if (report?.header != null) { + return report.header.glibcVersionRuntime == null; + } + } catch { + // fall through to the filesystem probe + } + // Fallback (older runtimes without process.report): look for the musl + // dynamic loader on disk. Default to glibc โ€” the common case โ€” when unsure. + try { + const fs = require("fs"); + return fs.readdirSync("/lib").some((f: string) => f.startsWith("ld-musl-")); + } catch { + return false; + } +} + +interface NativeResult { + error: string | null; + result: T | null; +} + +interface NativeAddon { + parseSync(query: string): NativeResult; + parsePlPgSQLSync(query: string): NativeResult; + fingerprintSync(query: string): NativeResult; + normalizeSync(query: string): NativeResult; + scanSync(query: string): NativeResult; +} + +const addon = loadNativeAddon(); + +function checkError(res: NativeResult): void { + if (res.error) { + const details: SqlErrorDetails = JSON.parse(res.error); + throw new SqlError(details.message, details); + } +} + +export function parseSync(query: string): ParseResult { + const res = addon.parseSync(query); + checkError(res); + return JSON.parse(res.result as string); +} + +export async function parse(query: string): Promise { + return parseSync(query); +} + +export function parsePlPgSQLSync(query: string): any { + const res = addon.parsePlPgSQLSync(query); + checkError(res); + return JSON.parse(res.result as string); +} + +export async function parsePlPgSQL(query: string): Promise { + return parsePlPgSQLSync(query); +} + +export function fingerprintSync(query: string): string { + const res = addon.fingerprintSync(query); + checkError(res); + return res.result as string; +} + +export async function fingerprint(query: string): Promise { + return fingerprintSync(query); +} + +export function normalizeSync(query: string): string { + const res = addon.normalizeSync(query); + checkError(res); + return res.result as string; +} + +export async function normalize(query: string): Promise { + return normalizeSync(query); +} + +export function scanSync(query: string): ScanResult { + const res = addon.scanSync(query); + checkError(res); + return res.result as ScanResult; +} + +export async function scan(query: string): Promise { + return scanSync(query); +} + +export async function loadModule(): Promise { + // no-op for native โ€” module is loaded synchronously via require() +} diff --git a/native/test/errors.test.js b/native/test/errors.test.js new file mode 100644 index 0000000..828d3a6 --- /dev/null +++ b/native/test/errors.test.js @@ -0,0 +1,123 @@ +const { parseSync, hasSqlDetails } = require("../dist/index.js"); +const { describe, it } = require("node:test"); +const assert = require("node:assert/strict"); + +describe("Error Handling", () => { + describe("Error Details Structure", () => { + it("should include sqlDetails property on parse errors", () => { + try { + parseSync("SELECT * FROM users WHERE id = @"); + assert.fail("Expected error"); + } catch (error) { + assert.ok("sqlDetails" in error); + assert.ok("message" in error.sqlDetails); + assert.ok("cursorPosition" in error.sqlDetails); + } + }); + + it("should have correct cursor position (0-based)", () => { + try { + parseSync("SELECT * FROM users WHERE id = @"); + assert.fail("Expected error"); + } catch (error) { + assert.equal(error.sqlDetails.cursorPosition, 32); + } + }); + + it("should identify error source file", () => { + try { + parseSync("SELECT * FROM users WHERE id = @"); + assert.fail("Expected error"); + } catch (error) { + assert.equal(error.sqlDetails.fileName, "scan.l"); + assert.equal(error.sqlDetails.functionName, "scanner_yyerror"); + } + }); + }); + + describe("Error Position Accuracy", () => { + const positionTests = [ + { query: "@ SELECT * FROM users", expectedPos: 0, desc: "error at start" }, + { query: "SELECT @ FROM users", expectedPos: 9, desc: "error after SELECT" }, + { query: "SELECT * FROM users WHERE id = @", expectedPos: 32, desc: "error at end" }, + ]; + + positionTests.forEach(({ query, expectedPos, desc }) => { + it(`should correctly identify position for ${desc}`, () => { + try { + parseSync(query); + assert.fail("Expected error"); + } catch (error) { + assert.equal(error.sqlDetails.cursorPosition, expectedPos); + } + }); + }); + }); + + describe("Error Types", () => { + it("should handle unterminated string literals", () => { + try { + parseSync("SELECT * FROM users WHERE name = 'unclosed"); + assert.fail("Expected error"); + } catch (error) { + assert.ok(error.message.includes("unterminated quoted string")); + } + }); + + it("should handle reserved keywords", () => { + try { + parseSync("SELECT * FROM table"); + assert.fail("Expected error"); + } catch (error) { + assert.ok(error.message.includes('syntax error at or near "table"')); + } + }); + }); + + describe("Edge Cases", () => { + it("should handle empty query", () => { + assert.throws(() => parseSync(""), { + message: "Query cannot be empty", + }); + }); + + it("should handle @ in strings", () => { + const query = "SELECT * FROM users WHERE email = 'user@example.com'"; + assert.doesNotThrow(() => parseSync(query)); + }); + }); + + describe("hasSqlDetails Type Guard", () => { + it("should return true for SQL parse errors", () => { + try { + parseSync("SELECT * FROM users WHERE id = @"); + assert.fail("Expected error"); + } catch (error) { + assert.equal(hasSqlDetails(error), true); + } + }); + + it("should return false for regular errors", () => { + assert.equal(hasSqlDetails(new Error("Regular error")), false); + }); + + it("should return false for non-Error objects", () => { + assert.equal(hasSqlDetails("string"), false); + assert.equal(hasSqlDetails(null), false); + assert.equal(hasSqlDetails(undefined), false); + }); + }); + + describe("Backward Compatibility", () => { + it("should maintain Error instance", () => { + try { + parseSync("SELECT * FROM users WHERE id = @"); + assert.fail("Expected error"); + } catch (error) { + assert.ok(error instanceof Error); + assert.ok(error.message); + assert.ok(error.stack); + } + }); + }); +}); diff --git a/native/test/fingerprint.test.js b/native/test/fingerprint.test.js new file mode 100644 index 0000000..cb3afcb --- /dev/null +++ b/native/test/fingerprint.test.js @@ -0,0 +1,57 @@ +const query = require("../dist/index.js"); +const { describe, it } = require("node:test"); +const assert = require("node:assert/strict"); + +describe("Query Fingerprinting", () => { + describe("Sync Fingerprinting", () => { + it("should return a fingerprint for a simple query", () => { + const fingerprint = query.fingerprintSync("select 1"); + assert.equal(typeof fingerprint, "string"); + assert.equal(fingerprint.length, 16); + }); + + it("should return same fingerprint for equivalent queries", () => { + const fp1 = query.fingerprintSync("select 1"); + const fp2 = query.fingerprintSync("SELECT 1"); + const fp3 = query.fingerprintSync("select 1"); + assert.equal(fp1, fp2); + assert.equal(fp1, fp3); + }); + + it("should return different fingerprints for different queries", () => { + const fp1 = query.fingerprintSync("select name from users"); + const fp2 = query.fingerprintSync("select id from customers"); + assert.notEqual(fp1, fp2); + }); + + it("should normalize parameter values", () => { + const fp1 = query.fingerprintSync("select * from users where id = 123"); + const fp2 = query.fingerprintSync("select * from users where id = 456"); + assert.equal(fp1, fp2); + }); + + it("should fail on invalid queries", () => { + assert.throws(() => query.fingerprintSync("NOT A QUERY"), Error); + }); + }); + + describe("Async Fingerprinting", () => { + it("should return a promise resolving to same result", async () => { + const testQuery = "select * from users"; + const fp = await query.fingerprint(testQuery); + assert.equal(fp, query.fingerprintSync(testQuery)); + }); + + it("should reject on bogus queries", async () => { + return query.fingerprint("NOT A QUERY").then( + () => { + throw new Error("should have rejected"); + }, + (e) => { + assert.ok(e instanceof Error); + assert.match(e.message, /NOT/); + } + ); + }); + }); +}); diff --git a/native/test/normalize.test.js b/native/test/normalize.test.js new file mode 100644 index 0000000..6f66584 --- /dev/null +++ b/native/test/normalize.test.js @@ -0,0 +1,61 @@ +const query = require("../dist/index.js"); +const { describe, it } = require("node:test"); +const assert = require("node:assert/strict"); + +describe("Query Normalization", () => { + describe("Sync Normalization", () => { + it("should normalize a simple query", () => { + const normalized = query.normalizeSync("select 1"); + assert.equal(typeof normalized, "string"); + assert.ok(normalized.includes("$1")); + }); + + it("should normalize parameter values", () => { + const n1 = query.normalizeSync("select * from users where id = 123"); + const n2 = query.normalizeSync("select * from users where id = 456"); + assert.equal(n1, n2); + assert.ok(n1.includes("$1")); + }); + + it("should normalize string literals", () => { + const n1 = query.normalizeSync("select * from users where name = 'john'"); + const n2 = query.normalizeSync("select * from users where name = 'jane'"); + assert.equal(n1, n2); + assert.ok(n1.includes("$1")); + }); + + it("should preserve query structure", () => { + const normalized = query.normalizeSync( + "SELECT id, name FROM users WHERE active = true ORDER BY name" + ); + assert.ok(normalized.includes("SELECT")); + assert.ok(normalized.includes("FROM")); + assert.ok(normalized.includes("WHERE")); + assert.ok(normalized.includes("ORDER BY")); + }); + + it("should fail on invalid queries", () => { + assert.throws(() => query.normalizeSync("NOT A QUERY"), Error); + }); + }); + + describe("Async Normalization", () => { + it("should return a promise resolving to same result", async () => { + const testQuery = "select * from users where id = 123"; + const normalized = await query.normalize(testQuery); + assert.equal(normalized, query.normalizeSync(testQuery)); + }); + + it("should reject on bogus queries", async () => { + return query.normalize("NOT A QUERY").then( + () => { + throw new Error("should have rejected"); + }, + (e) => { + assert.ok(e instanceof Error); + assert.match(e.message, /NOT/); + } + ); + }); + }); +}); diff --git a/native/test/parsing.test.js b/native/test/parsing.test.js new file mode 100644 index 0000000..e8d3b7c --- /dev/null +++ b/native/test/parsing.test.js @@ -0,0 +1,76 @@ +const query = require("../dist/index.js"); +const { describe, it } = require("node:test"); +const assert = require("node:assert/strict"); + +function removeLocationProperties(obj) { + if (typeof obj !== "object" || obj === null) return obj; + if (Array.isArray(obj)) return obj.map(removeLocationProperties); + const result = {}; + for (const key in obj) { + if (key === "location" || key === "stmt_len" || key === "stmt_location") + continue; + result[key] = removeLocationProperties(obj[key]); + } + return result; +} + +describe("Query Parsing", () => { + describe("Sync Parsing", () => { + it("should return a single-item parse result for common queries", () => { + const queries = ["select 1", "select null", "select ''", "select a, b"]; + const results = queries.map(query.parseSync); + results.forEach((res) => { + assert.equal(res.stmts.length, 1); + }); + + const selectedDatas = results.map( + (it) => it.stmts[0].stmt.SelectStmt.targetList + ); + + assert.equal(selectedDatas[0][0].ResTarget.val.A_Const.ival.ival, 1); + assert.equal(selectedDatas[1][0].ResTarget.val.A_Const.isnull, true); + assert.equal(selectedDatas[2][0].ResTarget.val.A_Const.sval.sval, ""); + assert.equal(selectedDatas[3].length, 2); + }); + + it("should support parsing multiple queries", () => { + const res = query.parseSync("select 1; select null;"); + assert.deepEqual( + res.stmts.map(removeLocationProperties), + [ + ...query.parseSync("select 1;").stmts.map(removeLocationProperties), + ...query + .parseSync("select null;") + .stmts.map(removeLocationProperties), + ] + ); + }); + + it("should not parse a bogus query", () => { + assert.throws(() => query.parseSync("NOT A QUERY"), Error); + }); + }); + + describe("Async parsing", () => { + it("should return a promise resolving to same result", async () => { + const testQuery = "select * from john;"; + const resPromise = query.parse(testQuery); + const res = await resPromise; + + assert.ok(resPromise instanceof Promise); + assert.deepEqual(res, query.parseSync(testQuery)); + }); + + it("should reject on bogus queries", async () => { + return query.parse("NOT A QUERY").then( + () => { + throw new Error("should have rejected"); + }, + (e) => { + assert.ok(e instanceof Error); + assert.match(e.message, /NOT/); + } + ); + }); + }); +}); diff --git a/native/test/plpgsql.test.js b/native/test/plpgsql.test.js new file mode 100644 index 0000000..5a4a81b --- /dev/null +++ b/native/test/plpgsql.test.js @@ -0,0 +1,65 @@ +const query = require("../dist/index.js"); +const { describe, it } = require("node:test"); +const assert = require("node:assert/strict"); + +describe("PL/pgSQL Parsing", () => { + describe("Sync PL/pgSQL Parsing", () => { + it("should parse a simple PL/pgSQL function", () => { + const funcSql = ` + CREATE OR REPLACE FUNCTION test_func() + RETURNS INTEGER AS $$ + BEGIN + RETURN 42; + END; + $$ LANGUAGE plpgsql; + `; + const result = query.parsePlPgSQLSync(funcSql); + assert.equal(typeof result, "object"); + assert.ok(Array.isArray(result.plpgsql_funcs)); + }); + + it("should parse function with parameters", () => { + const funcSql = ` + CREATE OR REPLACE FUNCTION add_numbers(a INTEGER, b INTEGER) + RETURNS INTEGER AS $$ + BEGIN + RETURN a + b; + END; + $$ LANGUAGE plpgsql; + `; + const result = query.parsePlPgSQLSync(funcSql); + assert.ok(result.plpgsql_funcs.length > 0); + }); + + it("should fail on invalid PL/pgSQL", () => { + assert.throws(() => query.parsePlPgSQLSync("NOT A FUNCTION"), Error); + }); + }); + + describe("Async PL/pgSQL Parsing", () => { + it("should return a promise resolving to same result", async () => { + const funcSql = ` + CREATE OR REPLACE FUNCTION test_func() + RETURNS INTEGER AS $$ + BEGIN + RETURN 42; + END; + $$ LANGUAGE plpgsql; + `; + const result = await query.parsePlPgSQL(funcSql); + assert.deepEqual(result, query.parsePlPgSQLSync(funcSql)); + }); + + it("should reject on invalid PL/pgSQL", async () => { + return query.parsePlPgSQL("NOT A FUNCTION").then( + () => { + throw new Error("should have rejected"); + }, + (e) => { + assert.ok(e instanceof Error); + assert.match(e.message, /NOT/); + } + ); + }); + }); +}); diff --git a/native/test/scan.test.js b/native/test/scan.test.js new file mode 100644 index 0000000..7143f30 --- /dev/null +++ b/native/test/scan.test.js @@ -0,0 +1,88 @@ +const query = require("../dist/index.js"); +const { describe, it } = require("node:test"); +const assert = require("node:assert/strict"); + +describe("Query Scanning", () => { + describe("Sync Scanning", () => { + it("should return a scan result with version and tokens", () => { + const result = query.scanSync("SELECT 1"); + assert.equal(typeof result, "object"); + assert.ok("version" in result); + assert.ok("tokens" in result); + assert.ok(Array.isArray(result.tokens)); + }); + + it("should scan a simple SELECT query correctly", () => { + const result = query.scanSync("SELECT 1"); + assert.equal(result.tokens.length, 2); + + const selectToken = result.tokens[0]; + assert.equal(selectToken.text, "SELECT"); + assert.equal(selectToken.start, 0); + assert.equal(selectToken.end, 6); + assert.equal(selectToken.keywordName, "RESERVED_KEYWORD"); + + const numberToken = result.tokens[1]; + assert.equal(numberToken.text, "1"); + assert.equal(numberToken.start, 7); + assert.equal(numberToken.end, 8); + assert.equal(numberToken.tokenName, "ICONST"); + assert.equal(numberToken.keywordName, "NO_KEYWORD"); + }); + + it("should scan tokens with correct positions", () => { + const sql = "SELECT * FROM users"; + const result = query.scanSync(sql); + assert.equal(result.tokens.length, 4); + result.tokens.forEach((token) => { + const actualText = sql.substring(token.start, token.end); + assert.equal(token.text, actualText); + }); + }); + + it("should identify different token types", () => { + const result = query.scanSync( + "SELECT 'string', 123, 3.14, $1 FROM users" + ); + const tokenTypes = result.tokens.map((t) => t.tokenName); + assert.ok(tokenTypes.includes("SCONST")); + assert.ok(tokenTypes.includes("ICONST")); + assert.ok(tokenTypes.includes("FCONST")); + assert.ok(tokenTypes.includes("PARAM")); + }); + + it("should handle complex queries with parameters", () => { + const result = query.scanSync( + "SELECT * FROM users WHERE id = $1 AND name = $2" + ); + const params = result.tokens.filter((t) => t.tokenName === "PARAM"); + assert.equal(params.length, 2); + assert.equal(params[0].text, "$1"); + assert.equal(params[1].text, "$2"); + }); + + it("should handle special PostgreSQL operators", () => { + const result = query.scanSync("SELECT id::text FROM users"); + const typecast = result.tokens.find((t) => t.text === "::"); + assert.ok(typecast); + assert.equal(typecast.tokenName, "TYPECAST"); + }); + + it("should preserve original token order", () => { + const result = query.scanSync( + "SELECT id, name FROM users ORDER BY name" + ); + for (let i = 1; i < result.tokens.length; i++) { + assert.ok(result.tokens[i].start >= result.tokens[i - 1].end); + } + }); + }); + + describe("Async Scanning", () => { + it("should return a promise resolving to same result as sync", async () => { + const testQuery = "SELECT * FROM users WHERE id = $1"; + const result = await query.scan(testQuery); + assert.deepEqual(result, query.scanSync(testQuery)); + }); + }); +}); diff --git a/native/test/smoke.mjs b/native/test/smoke.mjs new file mode 100644 index 0000000..3bdd591 --- /dev/null +++ b/native/test/smoke.mjs @@ -0,0 +1,100 @@ +/** + * Smoke test for the packaged native addon. + * Verifies that the platform package loads and all APIs work. + * + * Usage (from a temp directory with the packages installed): + * node smoke.mjs + * + * Or from the native/ directory with prebuilds in place: + * node test/smoke.mjs + */ + +import { createRequire } from "module"; + +const require = createRequire(import.meta.url); + +let exitCode = 0; + +function assert(condition, message) { + if (!condition) { + console.error(`FAIL: ${message}`); + exitCode = 1; + } +} + +async function test(name, fn) { + try { + await fn(); + console.log(` ok ${name}`); + } catch (e) { + console.error(` FAIL ${name}: ${e.message}`); + exitCode = 1; + } +} + +const lib = require("@ashbyhq/libpg-query-native"); + +console.log(`Platform: ${process.platform}-${process.arch}`); +console.log(`Node: ${process.version}`); +console.log(""); + +await test("parseSync returns parse tree", () => { + const result = lib.parseSync("SELECT 1"); + assert(result.version > 0, "version should be positive"); + assert(result.stmts.length === 1, "should have 1 statement"); + assert(result.stmts[0].stmt.SelectStmt, "should have SelectStmt"); +}); + +await test("parse (async) works", async () => { + const result = await lib.parse("SELECT 1"); + assert(result.stmts.length === 1, "should have 1 statement"); +}); + +await test("fingerprintSync returns hex string", () => { + const fp = lib.fingerprintSync("SELECT 1"); + assert(typeof fp === "string", "should be string"); + assert(fp.length === 16, "should be 16 chars"); + assert(/^[0-9a-f]+$/.test(fp), "should be hex"); +}); + +await test("normalizeSync replaces constants", () => { + const result = lib.normalizeSync("SELECT 1, 'hello'"); + assert(result.includes("$1"), "should have $1"); + assert(!result.includes("hello"), "should not have literal"); +}); + +await test("scanSync returns tokens", () => { + const result = lib.scanSync("SELECT id FROM users"); + assert(result.tokens.length > 0, "should have tokens"); + assert(result.tokens[0].text === "SELECT", "first token should be SELECT"); +}); + +await test("parsePlPgSQLSync works", () => { + const result = lib.parsePlPgSQLSync( + "CREATE FUNCTION test() RETURNS void AS $$ BEGIN NULL; END; $$ LANGUAGE plpgsql" + ); + assert(result.plpgsql_funcs, "should have plpgsql_funcs"); +}); + +await test("parseSync throws SqlError on bad SQL", () => { + try { + lib.parseSync("SELECTT"); + assert(false, "should have thrown"); + } catch (e) { + assert(e.name === "SqlError", `error name should be SqlError, got ${e.name}`); + assert(e.sqlDetails, "should have sqlDetails"); + assert(typeof e.sqlDetails.message === "string", "should have message"); + } +}); + +await test("loadModule is a no-op", async () => { + await lib.loadModule(); +}); + +console.log(""); +if (exitCode === 0) { + console.log("All smoke tests passed."); +} else { + console.error("Some smoke tests failed."); +} +process.exit(exitCode); diff --git a/native/tsconfig.json b/native/tsconfig.json new file mode 100644 index 0000000..3f3bd4a --- /dev/null +++ b/native/tsconfig.json @@ -0,0 +1,15 @@ +{ + "compilerOptions": { + "target": "ES2020", + "module": "commonjs", + "moduleResolution": "node", + "declaration": true, + "outDir": "dist", + "rootDir": "src", + "strict": true, + "esModuleInterop": true, + "skipLibCheck": true + }, + "include": ["src/**/*.ts"], + "exclude": ["node_modules", "dist", "test", "benchmark"] +}