diff --git a/docs/package.json b/docs/package.json index 5ce7a7181a8b..e69f4c890acb 100644 --- a/docs/package.json +++ b/docs/package.json @@ -3,7 +3,7 @@ "version": "0.0.0", "private": true, "scripts": { - "build": "yarn clean && yarn preprocess && yarn spellcheck && yarn preprocess:move && yarn validate:redirects && yarn validate:api-ref-links && docusaurus build && node scripts/append_api_docs_to_llms.js", + "build": "yarn clean && yarn preprocess && yarn spellcheck && yarn preprocess:move && yarn validate:redirects && yarn validate:api-ref-links && docusaurus build && node scripts/augment_sitemap.js && node scripts/append_api_docs_to_llms.js", "validate:redirects": "./scripts/validate_redirect_targets.sh", "validate:api-ref-links": "./scripts/validate_api_ref_links.sh", "clean": "./scripts/clean.sh", diff --git a/docs/scripts/append_api_docs_to_llms.js b/docs/scripts/append_api_docs_to_llms.js index 79f4c40e8426..602bff3486da 100644 --- a/docs/scripts/append_api_docs_to_llms.js +++ b/docs/scripts/append_api_docs_to_llms.js @@ -38,10 +38,19 @@ if (defaultType && fs.existsSync(path.join(STATIC_DIR, `aztec-nr-api/${defaultTy name: "Aztec.nr API Reference", dir: `aztec-nr-api/${defaultType}`, description: `Auto-generated API documentation for Aztec.nr (${defaultVersion})`, + format: "html", }); } else if (!defaultType) { console.warn("Warning: No default version found for API docs"); } +if (defaultType && fs.existsSync(path.join(STATIC_DIR, `typescript-api/${defaultType}`))) { + API_DIRS.push({ + name: "TypeScript API Reference", + dir: `typescript-api/${defaultType}`, + description: `Auto-generated TypeScript API documentation for Aztec packages (${defaultVersion})`, + format: "markdown", + }); +} /** * Extract text content from HTML, stripping tags and normalizing whitespace. @@ -94,9 +103,9 @@ function htmlToText(html) { } /** - * Recursively find all HTML files in a directory. + * Recursively find all files with a given extension in a directory. */ -function findHtmlFiles(dir, files = []) { +function findFiles(dir, ext, files = []) { if (!fs.existsSync(dir)) { return files; } @@ -106,8 +115,8 @@ function findHtmlFiles(dir, files = []) { for (const entry of entries) { const fullPath = path.join(dir, entry.name); if (entry.isDirectory()) { - findHtmlFiles(fullPath, files); - } else if (entry.name.endsWith(".html")) { + findFiles(fullPath, ext, files); + } else if (entry.name.endsWith(ext)) { files.push(fullPath); } } @@ -115,6 +124,21 @@ function findHtmlFiles(dir, files = []) { return files; } +/** + * Recursively find all HTML files in a directory. + */ +function findHtmlFiles(dir) { + return findFiles(dir, ".html"); +} + +/** + * Recursively find all markdown files in a directory. + * Note: `llm-summary.txt` is naturally excluded since it does not end in `.md`. + */ +function findMarkdownFiles(dir) { + return findFiles(dir, ".md"); +} + /** * Get the relative URL path for a file. */ @@ -182,10 +206,14 @@ function main() { continue; } - const htmlFiles = sortByImportance(findHtmlFiles(dirPath)); - console.log(`Found ${htmlFiles.length} HTML files in ${apiDir.dir}`); + const isMarkdown = apiDir.format === "markdown"; + const files = isMarkdown + ? findMarkdownFiles(dirPath) + : sortByImportance(findHtmlFiles(dirPath)); + const ext = isMarkdown ? ".md" : ".html"; + console.log(`Found ${files.length} ${isMarkdown ? "markdown" : "HTML"} files in ${apiDir.dir}`); - if (htmlFiles.length === 0) { + if (files.length === 0) { continue; } @@ -195,30 +223,43 @@ function main() { fullContentSection += `## ${apiDir.name}\n\n`; fullContentSection += `${apiDir.description}\n\n`; - // Process only index files for links to avoid overwhelming the llms.txt - const indexFiles = htmlFiles.filter( - (f) => f.endsWith("index.html") || f.includes("/fn.") || f.includes("/struct.") || f.includes("/trait.") - ); - - // Add links for key files - for (const file of indexFiles.slice(0, 100)) { - // Limit to 100 links per section - const urlPath = getUrlPath(file, STATIC_DIR); - const fileName = path.basename(file, ".html"); - linksSection += `- [${fileName}](${urlPath})\n`; - } + if (isMarkdown) { + // For markdown API docs, add a link per file and include llm-summary.txt if present + const summaryPath = path.join(dirPath, "llm-summary.txt"); + if (fs.existsSync(summaryPath)) { + linksSection += fs.readFileSync(summaryPath, "utf-8") + "\n\n"; + } + for (const file of files) { + const urlPath = getUrlPath(file, STATIC_DIR); + const fileName = path.basename(file, ext); + linksSection += `- [${fileName}](${urlPath})\n`; + } + } else { + // For HTML API docs, process only index files for links + const indexFiles = files.filter( + (f) => f.endsWith("index.html") || f.includes("/fn.") || f.includes("/struct.") || f.includes("/trait.") + ); + + // Add links for key files + for (const file of indexFiles.slice(0, 100)) { + // Limit to 100 links per section + const urlPath = getUrlPath(file, STATIC_DIR); + const fileName = path.basename(file, ext); + linksSection += `- [${fileName}](${urlPath})\n`; + } - if (indexFiles.length > 100) { - linksSection += `- ... and ${indexFiles.length - 100} more files\n`; + if (indexFiles.length > 100) { + linksSection += `- ... and ${indexFiles.length - 100} more files\n`; + } } linksSection += "\n"; // Add full content for all files - for (const file of htmlFiles) { + for (const file of files) { try { - const html = fs.readFileSync(file, "utf-8"); - const text = htmlToText(html); + const raw = fs.readFileSync(file, "utf-8"); + const text = isMarkdown ? raw.trim() : htmlToText(raw); if (text.length > 100) { // Only include if there's meaningful content diff --git a/docs/scripts/augment_sitemap.js b/docs/scripts/augment_sitemap.js new file mode 100644 index 000000000000..f3df45a23fe2 --- /dev/null +++ b/docs/scripts/augment_sitemap.js @@ -0,0 +1,102 @@ +#!/usr/bin/env node +/** + * Post-build script to add static API documentation URLs to the sitemap. + * + * Docusaurus only includes its managed routes in sitemap.xml. This script + * appends entries for the auto-generated API docs in static/ that are + * copied to build/ but not indexed by the sitemap plugin. + */ + +const fs = require("fs"); +const path = require("path"); + +const BUILD_DIR = path.join(__dirname, "..", "build"); +// Override with SITE_URL env var if the canonical URL ever changes. +// Kept in sync with `url` in docusaurus.config.js. +const SITE_URL = process.env.SITE_URL || "https://docs.aztec.network"; + +// Load version config to determine which version subdirectory to index. +let developerVersionConfig; +try { + developerVersionConfig = require("../developer_version_config.json"); +} catch { + developerVersionConfig = null; +} + +const defaultType = developerVersionConfig?.mainnet + ? "mainnet" + : developerVersionConfig?.testnet + ? "testnet" + : null; + +if (!defaultType) { + console.warn("Warning: No default version found — skipping sitemap augmentation"); + process.exit(0); +} + +/** + * Recursively find all files with a given extension. + */ +function findFiles(dir, ext) { + const results = []; + if (!fs.existsSync(dir)) return results; + + const entries = fs.readdirSync(dir, { withFileTypes: true }); + for (const entry of entries) { + const fullPath = path.join(dir, entry.name); + if (entry.isDirectory()) { + results.push(...findFiles(fullPath, ext)); + } else if (entry.name.endsWith(ext)) { + results.push(fullPath); + } + } + return results; +} + +function main() { + const sitemapPath = path.join(BUILD_DIR, "sitemap.xml"); + + if (!fs.existsSync(sitemapPath)) { + console.error("Error: build/sitemap.xml not found. Run the build first."); + process.exit(1); + } + + let sitemap = fs.readFileSync(sitemapPath, "utf-8"); + + // Aztec.nr API HTML files (skip raw markdown — those aren't browsable pages) + const nrApiDir = path.join(BUILD_DIR, `aztec-nr-api/${defaultType}`); + // Exclude Noir stdlib (duplicated at noir-lang.org), the all.html mega-index, + // and per-constant global.*.html pages. Keeps the sitemap aligned with the + // Typesense stop_urls list so both discovery paths surface the same content. + const EXCLUDE_RE = new RegExp( + `aztec-nr-api/${defaultType}/(std/|all\\.html$|.*/global\\.[^/]+\\.html$)` + ); + const htmlFiles = findFiles(nrApiDir, ".html").filter( + (f) => !EXCLUDE_RE.test(f.replace(/\\/g, "/")) + ); + + if (htmlFiles.length === 0) { + console.log("No static API docs found to add to sitemap"); + return; + } + + // Build XML entries + const entries = htmlFiles + .map((file) => { + const relativePath = path.relative(BUILD_DIR, file).replace(/\\/g, "/"); + return `${SITE_URL}/${relativePath}monthly0.3`; + }) + .join(""); + + // Insert before closing + if (!sitemap.includes("")) { + console.error("Error: build/sitemap.xml missing closing tag — aborting."); + process.exit(1); + } + sitemap = sitemap.replace("", entries + ""); + + fs.writeFileSync(sitemapPath, sitemap); + console.log(`Added ${htmlFiles.length} Aztec.nr API doc URLs to sitemap.xml`); +} + +main(); diff --git a/docs/typesense.config.json b/docs/typesense.config.json index 7dccc1f3dc63..dda3415cd9f6 100644 --- a/docs/typesense.config.json +++ b/docs/typesense.config.json @@ -1,26 +1,52 @@ { "index_name": "aztec-docs", "start_urls": [ - "https://docs.aztec.network/" + { + "url": "https://docs.aztec.network/", + "page_rank": 10 + }, + { + "url": "https://docs.aztec.network/aztec-nr-api/mainnet/", + "selectors_key": "api-nr", + "page_rank": 5 + } ], "sitemap_urls": [ "https://docs.aztec.network/sitemap.xml" ], + "stop_urls": [ + "https://docs.aztec.network/aztec-nr-api/mainnet/std/", + "https://docs.aztec.network/aztec-nr-api/mainnet/all.html", + "aztec-nr-api/.*/global\\.[^/]+\\.html$" + ], "sitemap_alternate_links": true, "selectors": { - "lvl0": { - "selector": "(//ul[contains(@class,'menu__list')]//a[contains(@class, 'menu__link menu__link--sublist menu__link--active')]/text() | //nav[contains(@class, 'navbar')]//a[contains(@class, 'navbar__link--active')]/text())[last()]", - "type": "xpath", - "global": true, - "default_value": "Documentation" + "default": { + "lvl0": { + "selector": "(//ul[contains(@class,'menu__list')]//a[contains(@class, 'menu__link menu__link--sublist menu__link--active')]/text() | //nav[contains(@class, 'navbar')]//a[contains(@class, 'navbar__link--active')]/text())[last()]", + "type": "xpath", + "global": true, + "default_value": "Documentation" + }, + "lvl1": "header h1", + "lvl2": "header h2", + "lvl3": "header h3", + "lvl4": "header h4", + "lvl5": "header h5", + "lvl6": "header h6", + "text": "article p, article li, article td:last-child" }, - "lvl1": "header h1", - "lvl2": "header h2", - "lvl3": "header h3", - "lvl4": "header h4", - "lvl5": "header h5", - "lvl6": "header h6", - "text": "article p, article li, article td:last-child" + "api-nr": { + "lvl0": { + "selector": "nav.sidebar h1 a", + "default_value": "Aztec.nr API Reference" + }, + "lvl1": "main h1", + "lvl2": "main h2", + "lvl3": "main h3", + "lvl4": "main h4", + "text": "main .comments p, main .item-description, main li, main pre code" + } }, "strip_chars": " .,;:#", "custom_settings": {