feat(guardrails): run PII via Presidio sidecars + TS recognizer registry

- replace the per-call python3 subprocess (cold spaCy load every call) with
  two long-lived Presidio sidecars (analyzer + anonymizer) reached over HTTP;
  the app image no longer carries Python/Presidio/venv
- add PRESIDIO_ANALYZER_URL / PRESIDIO_ANONYMIZER_URL
- move VIN out of Python into a TS recognizer (check-digit validated) behind a
  CUSTOM_RECOGNIZERS registry so new custom detectors are one entry; masking is
  handled uniformly by the anonymizer
- drive the guardrails block's PII type picker from the shared pii-entities
  catalog (adds VIN, fixes drift) so block + Data Retention never diverge
- delete validate_pii.py, requirements.txt, setup.sh and the Dockerfile venv step

Author: TheodoreSpeaks

apps/sim/blocks/blocks/guardrails.ts

@@ -1,4 +1,5 @@
 import { ShieldCheckIcon } from '@/components/icons'
+import { PII_ENTITY_GROUPS } from '@/lib/guardrails/pii-entities'
 import type { BlockConfig } from '@/blocks/types'
 import {
   getModelOptions,
@@ -170,65 +171,15 @@ Return ONLY the regex pattern - no explanations, no quotes, no forward slashes,
       title: 'PII Types to Detect',
       type: 'grouped-checkbox-list',
       maxHeight: 400,
-      options: [
-        // Common PII types
-        { label: 'Person name', id: 'PERSON', group: 'Common' },
-        { label: 'Email address', id: 'EMAIL_ADDRESS', group: 'Common' },
-        { label: 'Phone number', id: 'PHONE_NUMBER', group: 'Common' },
-        { label: 'Location', id: 'LOCATION', group: 'Common' },
-        { label: 'Date or time', id: 'DATE_TIME', group: 'Common' },
-        { label: 'IP address', id: 'IP_ADDRESS', group: 'Common' },
-        { label: 'URL', id: 'URL', group: 'Common' },
-        { label: 'Credit card number', id: 'CREDIT_CARD', group: 'Common' },
-        { label: 'International bank account number (IBAN)', id: 'IBAN_CODE', group: 'Common' },
-        { label: 'Cryptocurrency wallet address', id: 'CRYPTO', group: 'Common' },
-        { label: 'Medical license number', id: 'MEDICAL_LICENSE', group: 'Common' },
-        { label: 'Nationality / religion / political group', id: 'NRP', group: 'Common' },
-
-        // USA
-        { label: 'US bank account number', id: 'US_BANK_NUMBER', group: 'USA' },
-        { label: 'US driver license number', id: 'US_DRIVER_LICENSE', group: 'USA' },
-        {
-          label: 'US individual taxpayer identification number (ITIN)',
-          id: 'US_ITIN',
-          group: 'USA',
-        },
-        { label: 'US passport number', id: 'US_PASSPORT', group: 'USA' },
-        { label: 'US Social Security number', id: 'US_SSN', group: 'USA' },
-
-        // UK
-        { label: 'UK National Insurance number', id: 'UK_NINO', group: 'UK' },
-        { label: 'UK NHS number', id: 'UK_NHS', group: 'UK' },
-
-        // Spain
-        { label: 'Spanish NIF number', id: 'ES_NIF', group: 'Spain' },
-        { label: 'Spanish NIE number', id: 'ES_NIE', group: 'Spain' },
-
-        // Italy
-        { label: 'Italian fiscal code', id: 'IT_FISCAL_CODE', group: 'Italy' },
-        { label: 'Italian driver license', id: 'IT_DRIVER_LICENSE', group: 'Italy' },
-        { label: 'Italian identity card', id: 'IT_IDENTITY_CARD', group: 'Italy' },
-        { label: 'Italian passport', id: 'IT_PASSPORT', group: 'Italy' },
-
-        // Poland
-        { label: 'Polish PESEL', id: 'PL_PESEL', group: 'Poland' },
-
-        // Singapore
-        { label: 'Singapore NRIC/FIN', id: 'SG_NRIC_FIN', group: 'Singapore' },
-
-        // Australia
-        { label: 'Australian business number (ABN)', id: 'AU_ABN', group: 'Australia' },
-        { label: 'Australian company number (ACN)', id: 'AU_ACN', group: 'Australia' },
-        { label: 'Australian tax file number (TFN)', id: 'AU_TFN', group: 'Australia' },
-        { label: 'Australian Medicare number', id: 'AU_MEDICARE', group: 'Australia' },
-
-        // India
-        { label: 'Indian Aadhaar', id: 'IN_AADHAAR', group: 'India' },
-        { label: 'Indian PAN', id: 'IN_PAN', group: 'India' },
-        { label: 'Indian vehicle registration', id: 'IN_VEHICLE_REGISTRATION', group: 'India' },
-        { label: 'Indian voter number', id: 'IN_VOTER', group: 'India' },
-        { label: 'Indian passport', id: 'IN_PASSPORT', group: 'India' },
-      ],
+      // Driven by the shared catalog (includes VIN and custom recognizers) so the
+      // block and the Data Retention settings never drift.
+      options: PII_ENTITY_GROUPS.flatMap((group) =>
+        group.entities.map((entity) => ({
+          label: entity.label,
+          id: entity.value,
+          group: group.label,
+        }))
+      ),
       condition: {
         field: 'validationType',
         value: ['pii'],

apps/sim/lib/core/config/env.ts

@@ -311,6 +311,8 @@ export const env = createEnv({
     PORT:                                  z.number().optional(),                  // Main application port
     INTERNAL_API_BASE_URL:                 z.string().optional(),                  // Optional internal base URL for server-side self-calls; must include protocol if set (e.g., http://sim-app.namespace.svc.cluster.local:3000)
     ALLOWED_ORIGINS:                       z.string().optional(),                  // CORS allowed origins
+    PRESIDIO_ANALYZER_URL:                 z.string().optional(),                  // Presidio analyzer sidecar base URL for PII detection (default http://localhost:5002)
+    PRESIDIO_ANONYMIZER_URL:               z.string().optional(),                  // Presidio anonymizer sidecar base URL for PII masking (default http://localhost:5001)
 
     // OAuth Integration Credentials - All optional, enables third-party integrations
     GOOGLE_CLIENT_ID:                      z.string().optional(),                  // Google OAuth client ID for Google services

apps/sim/lib/guardrails/.gitignore

@@ -19,22 +19,26 @@ For **hallucination detection**, you'll need:
 - A knowledge base with documents
 - An LLM provider API key (or use hosted models)
 
-### Python Validators (PII Detection)
+### PII Detection (Presidio sidecars)
 
-For **PII detection**, you need to set up a Python virtual environment and install Microsoft Presidio:
+PII detection runs against two long-lived **Microsoft Presidio sidecar containers** reached over
+HTTP — the analyzer (NLP detection) and the anonymizer (masking). In deployment they run alongside the
+app container in the same ECS task; locally, run the official images:
 
 ```bash
-cd apps/sim/lib/guardrails
-./setup.sh
+docker run -d -p 5002:3000 mcr.microsoft.com/presidio-analyzer:latest
+docker run -d -p 5001:3000 mcr.microsoft.com/presidio-anonymizer:latest
 ```
 
-This will:
-1. Create a Python virtual environment in `apps/sim/lib/guardrails/venv`
-2. Install required dependencies:
-   - `presidio-analyzer` - PII detection engine
-   - `presidio-anonymizer` - PII masking/anonymization
+Point the app at them (defaults shown):
 
-The TypeScript wrapper will automatically use the virtual environment's Python interpreter.
+```bash
+PRESIDIO_ANALYZER_URL=http://localhost:5002
+PRESIDIO_ANONYMIZER_URL=http://localhost:5001
+```
+
+VIN recognition (check-digit validated) is implemented in TypeScript (`vin.ts`) and never sent to the
+sidecars. No Python or local venv is required.
 
 ## Usage
 
@@ -93,10 +97,9 @@ See [Presidio documentation](https://microsoft.github.io/presidio/supported_enti
 - `validate_json.ts` - JSON validation (TypeScript)
 - `validate_regex.ts` - Regex validation (TypeScript)
 - `validate_hallucination.ts` - Hallucination detection with RAG + LLM scoring (TypeScript)
-- `validate_pii.ts` - PII detection TypeScript wrapper (TypeScript)
-- `validate_pii.py` - PII detection using Microsoft Presidio (Python)
+- `validate_pii.ts` - PII detection client: calls the Presidio analyzer/anonymizer sidecars (TypeScript)
+- `vin.ts` - Check-digit-validated VIN recognizer (TypeScript)
+- `pii-entities.ts` - Client-safe PII entity catalog
+- `mask-client.ts` - Internal HTTP client for batch PII masking from the log-redaction persist path
 - `validate.test.ts` - Test suite for JSON and regex validators
-- `validate_hallucination.py` - Legacy Python hallucination detector (deprecated)
-- `requirements.txt` - Python dependencies for PII detection (and legacy hallucination)
-- `setup.sh` - Legacy installation script (deprecated)
 

apps/sim/lib/guardrails/README.md

@@ -0,0 +1,27 @@
+import { findVins } from '@/lib/guardrails/vin'
+
+/**
+ * A custom PII recognizer for entities Microsoft Presidio doesn't ship.
+ *
+ * A recognizer only does **detection** — it returns character spans. Masking is
+ * handled uniformly by the anonymizer sidecar, which replaces every span by its
+ * `entityType` (e.g. `<VIN>`), so a recognizer never touches the sidecars or the
+ * masking path.
+ *
+ * To add one:
+ * 1. Implement a pure `detect(text)` (regex/checksum/etc., no I/O).
+ * 2. Register it in {@link CUSTOM_RECOGNIZERS}.
+ * 3. Add its entity to `pii-entities.ts` so it appears in the Data Retention UI.
+ */
+export interface CustomRecognizer {
+  /** Entity name; becomes the `<ENTITY>` placeholder when masked. */
+  entityType: string
+  /** Character spans of confirmed matches in `text`. Pure — no I/O. */
+  detect(text: string): Array<{ start: number; end: number }>
+}
+
+/** The registry of TS-side recognizers, applied on top of Presidio's built-ins. */
+export const CUSTOM_RECOGNIZERS: CustomRecognizer[] = [{ entityType: 'VIN', detect: findVins }]
+
+/** Entity names owned by a custom recognizer — never forwarded to the analyzer. */
+export const CUSTOM_ENTITY_TYPES = new Set(CUSTOM_RECOGNIZERS.map((r) => r.entityType))