fix(reference): replace hand-rolled HTML stripping with turndown

Author: ericallam

pnpm-lock.yaml

@@ -24,6 +24,7 @@
     "react": "^19.0.0",
     "react-dom": "^19.0.0",
     "streamdown": "^2.3.0",
+    "turndown": "^7.2.2",
     "zod": "3.25.76"
   },
   "devDependencies": {
@@ -32,6 +33,7 @@
     "@types/node": "^22",
     "@types/react": "^19",
     "@types/react-dom": "^19",
+    "@types/turndown": "^5.0.6",
     "tailwindcss": "^4",
     "prisma": "^7.4.2",
     "trigger.dev": "workspace:*",

references/ai-chat/package.json

@@ -12,8 +12,11 @@ import { PrismaClient } from "../../lib/generated/prisma/client";
 const adapter = new PrismaPg({ connectionString: process.env.DATABASE_URL! });
 const prisma = new PrismaClient({ adapter });
 
+import TurndownService from "turndown";
 import { DEFAULT_MODEL, REASONING_MODELS } from "@/lib/models";
 
+const turndown = new TurndownService();
+
 const MODELS: Record<string, () => LanguageModel> = {
   "gpt-4o-mini": () => openai("gpt-4o-mini"),
   "gpt-4o": () => openai("gpt-4o"),
@@ -98,20 +101,8 @@ const webFetch = tool({
     let text = await response.text();
     const contentType = response.headers.get("content-type") ?? "";
 
-    // Strip HTML to plain text for readability
     if (contentType.includes("html")) {
-      text = text
-        .replace(/<script[\s\S]*?<\/script>/gi, "")
-        .replace(/<style[\s\S]*?<\/style>/gi, "")
-        .replace(/<[^>]+>/g, " ")
-        .replace(/&nbsp;/g, " ")
-        .replace(/&amp;/g, "&")
-        .replace(/&lt;/g, "<")
-        .replace(/&gt;/g, ">")
-        .replace(/&quot;/g, '"')
-        .replace(/&#39;/g, "'")
-        .replace(/\s+/g, " ")
-        .trim();
+      text = turndown.turndown(text);
     }
 
     return {
@@ -204,16 +195,7 @@ export const deepResearch = schemaTask({
         const contentType = response.headers.get("content-type") ?? "";
 
         if (contentType.includes("html")) {
-          text = text
-            .replace(/<script[\s\S]*?<\/script>/gi, "")
-            .replace(/<style[\s\S]*?<\/style>/gi, "")
-            .replace(/<[^>]+>/g, " ")
-            .replace(/&nbsp;/g, " ")
-            .replace(/&amp;/g, "&")
-            .replace(/&lt;/g, "<")
-            .replace(/&gt;/g, ">")
-            .replace(/\s+/g, " ")
-            .trim();
+          text = turndown.turndown(text);
         }
 
         results.push({