feat: parallelization, HTML report overhaul, Zig parser, dynamic test hardening

.gitignore

@@ -5,4 +5,5 @@ __pycache__/
 .venv/
 node_modules/
 apps/openant-cli/bin/
-docs/
+libs/openant-core/parsers/go/go_parser/go_parser
+# docs/

CHANGELOG.md

@@ -0,0 +1,42 @@
+# Changelog
+
+All notable changes to OpenAnt are documented in this file.
+
+## [Unreleased]
+
+This release syncs a large body of work from internal development. Highlights:
+
+### Added
+
+- **Parallelization** across all pipeline stages:
+  - Stage 1 analysis (Detect), Stage 2 verification, Enhance, and Dynamic Test now run units concurrently via worker pools.
+  - Thread-safe `TokenTracker` and `ProgressReporter` for correct aggregate metrics under parallel execution.
+  - Shared HTTP client and a token-bucket `RateLimiter` (`libs/openant-core/utilities/rate_limiter.py`) to stay within Anthropic API rate limits.
+- **Checkpoint / resume system** (`libs/openant-core/core/checkpoint.py`): every phase persists per-unit progress so interrupted scans can resume without re-running completed work.
+- **Zig parser** (`libs/openant-core/parsers/zig/`): repository scanner, unit generator, and test pipeline.
+- **HTML report improvements** (`apps/openant-cli/internal/report/`):
+  - Two themes: dark (`overview.gohtml`) and Knostic-branded light (`report-reskin.gohtml`).
+  - Report header shows repo name, commit SHA, language, total scan duration (formatted `Xd Xh Xm Xs`), and cost.
+  - Findings are numbered (`#N`), have anchor IDs, and are grouped into collapsible sections by verdict (vulnerable / bypassable open by default; inconclusive / protected / safe closed).
+  - Within each verdict group, findings are sub-sorted by dynamic test outcome (CONFIRMED first, NOT_REPRODUCED last).
+  - File paths link directly to the repo at the exact commit.
+  - Pipeline Costs & Timing section with per-step breakdown and a Totals row.
+  - Executive Summary links to findings via `#N` references; priority labels (Critical / High / Medium) replace fabricated timeframes.
+- **Dynamic testing** hardening: structured result classification (CONFIRMED / NOT_REPRODUCED / BLOCKED / INCONCLUSIVE / ERROR), Docker template updates, retry logic, and checkpoint-aware resume.
+- `openant build-output` and `openant dynamic-test` subcommands with prompt-before-skip UX.
+
+### Changed
+
+- Finding verifier (`utilities/finding_verifier.py`) hardened with better error handling and agentic tool integration.
+- Context enhancer (`utilities/context_enhancer.py`) overhauled for parallel, agentic enhancement.
+- Report data pipeline rewritten: Python computes a `ReportData` JSON blob; Go renders the HTML template.
+- Cost tracking reworked to report per-unit costs in progress output and aggregate correctly across parallel workers.
+
+### Fixed
+
+- Cost tracking no longer shows negative or incorrect totals under parallel execution.
+- `merge_dynamic_results` no longer contaminates stdout, unblocking clean JSON output.
+- HTML report entities (`>`, `<`) render correctly (previously double-escaped).
+- "Max iterations reached" verifier timeouts now mark findings as `inconclusive` rather than leaving a stale verdict.
+- Checkpoint resume behavior unified across phases.
+- Stdin race during interactive signal forwarding.

apps/openant-cli/cmd/analyze.go

@@ -4,6 +4,7 @@ import (
 	"fmt"
 	"os"
 
+	"github.com/knostic/open-ant-cli/internal/checkpoint"
 	"github.com/knostic/open-ant-cli/internal/output"
 	"github.com/knostic/open-ant-cli/internal/python"
 	"github.com/spf13/cobra"
@@ -31,6 +32,9 @@ var (
 	analyzeExploitOnly    bool
 	analyzeLimit          int
 	analyzeModel          string
+	analyzeWorkers        int
+	analyzeCheckpoint     string
+	analyzeBackoff        int
 )
 
 func init() {
@@ -42,6 +46,9 @@ func init() {
 	analyzeCmd.Flags().BoolVar(&analyzeExploitOnly, "exploitable-only", false, "Only analyze units classified as exploitable by enhancer")
 	analyzeCmd.Flags().IntVar(&analyzeLimit, "limit", 0, "Max units to analyze (0 = no limit)")
 	analyzeCmd.Flags().StringVar(&analyzeModel, "model", "opus", "Model: opus or sonnet")
+	analyzeCmd.Flags().IntVar(&analyzeWorkers, "workers", 8, "Number of parallel workers for LLM steps (default: 8)")
+	analyzeCmd.Flags().StringVar(&analyzeCheckpoint, "checkpoint", "", "Path to checkpoint directory for save/resume")
+	analyzeCmd.Flags().IntVar(&analyzeBackoff, "backoff", 30, "Seconds to wait when rate-limited (default: 30)")
 }
 
 func runAnalyze(cmd *cobra.Command, args []string) {
@@ -74,6 +81,17 @@ func runAnalyze(cmd *cobra.Command, args []string) {
 		os.Exit(2)
 	}
 
+	// Auto-detect checkpoints from a previous interrupted run
+	if analyzeCheckpoint == "" && ctx != nil {
+		if cpInfo := checkpoint.DetectViaPython(rt.Path, ctx.ScanDir, "analyze"); cpInfo != nil {
+			if checkpoint.PromptResume(cpInfo, "analyze", quiet) {
+				analyzeCheckpoint = cpInfo.Dir
+			} else {
+				_ = checkpoint.Clean(cpInfo.Dir)
+			}
+		}
+	}
+
 	pyArgs := []string{"analyze", datasetPath, "--output", analyzeOutput}
 	if analyzeVerify {
 		pyArgs = append(pyArgs, "--verify")
@@ -96,14 +114,25 @@ func runAnalyze(cmd *cobra.Command, args []string) {
 	if analyzeModel != "opus" {
 		pyArgs = append(pyArgs, "--model", analyzeModel)
 	}
+	if analyzeWorkers != 8 {
+		pyArgs = append(pyArgs, "--workers", fmt.Sprintf("%d", analyzeWorkers))
+	}
+	if analyzeCheckpoint != "" {
+		pyArgs = append(pyArgs, "--checkpoint", analyzeCheckpoint)
+	}
+	if analyzeBackoff != 30 {
+		pyArgs = append(pyArgs, "--backoff", fmt.Sprintf("%d", analyzeBackoff))
+	}
 
 	result, err := python.Invoke(rt.Path, pyArgs, "", quiet, requireAPIKey())
 	if err != nil {
 		output.PrintError(err.Error())
 		os.Exit(2)
 	}
 
-	if jsonOutput {
+	if result.Envelope.Status == "interrupted" {
+		os.Exit(130)
+	} else if jsonOutput {
 		output.PrintJSON(result.Envelope)
 	} else if result.Envelope.Status == "success" {
 		if data, ok := result.Envelope.Data.(map[string]any); ok {

apps/openant-cli/cmd/dynamictest.go

@@ -4,6 +4,7 @@ import (
 	"fmt"
 	"os"
 
+	"github.com/knostic/open-ant-cli/internal/checkpoint"
 	"github.com/knostic/open-ant-cli/internal/output"
 	"github.com/knostic/open-ant-cli/internal/python"
 	"github.com/spf13/cobra"
@@ -41,6 +42,12 @@ func runDynamicTest(cmd *cobra.Command, args []string) {
 		os.Exit(2)
 	}
 
+	// Check pipeline_output.json exists before launching Python
+	if _, err := os.Stat(pipelineOutputPath); err != nil {
+		output.PrintError("pipeline_output.json not found. Run 'openant build-output' first.")
+		os.Exit(2)
+	}
+
 	// Apply project defaults
 	if ctx != nil {
 		if dynamicTestOutput == "" {
@@ -54,6 +61,17 @@ func runDynamicTest(cmd *cobra.Command, args []string) {
 		os.Exit(2)
 	}
 
+	// Auto-detect checkpoints from a previous interrupted run
+	if ctx != nil {
+		if cpInfo := checkpoint.DetectViaPython(rt.Path, ctx.ScanDir, "dynamic_test"); cpInfo != nil {
+			if checkpoint.PromptResume(cpInfo, "dynamic-test", quiet) {
+				// Resume — Python auto-detects checkpoint dir in output dir
+			} else {
+				_ = checkpoint.Clean(cpInfo.Dir)
+			}
+		}
+	}
+
 	pyArgs := []string{"dynamic-test", pipelineOutputPath}
 	if dynamicTestOutput != "" {
 		pyArgs = append(pyArgs, "--output", dynamicTestOutput)
@@ -68,7 +86,9 @@ func runDynamicTest(cmd *cobra.Command, args []string) {
 		os.Exit(2)
 	}
 
-	if jsonOutput {
+	if result.Envelope.Status == "interrupted" {
+		os.Exit(130)
+	} else if jsonOutput {
 		output.PrintJSON(result.Envelope)
 	} else if result.Envelope.Status == "success" {
 		if data, ok := result.Envelope.Data.(map[string]any); ok {

apps/openant-cli/cmd/enhance.go

@@ -1,8 +1,10 @@
 package cmd
 
 import (
+	"fmt"
 	"os"
 
+	"github.com/knostic/open-ant-cli/internal/checkpoint"
 	"github.com/knostic/open-ant-cli/internal/output"
 	"github.com/knostic/open-ant-cli/internal/python"
 	"github.com/spf13/cobra"
@@ -28,6 +30,8 @@ var (
 	enhanceRepoPath       string
 	enhanceMode           string
 	enhanceCheckpoint     string
+	enhanceWorkers        int
+	enhanceBackoff        int
 )
 
 func init() {
@@ -36,6 +40,8 @@ func init() {
 	enhanceCmd.Flags().StringVar(&enhanceRepoPath, "repo-path", "", "Path to the repository (required for agentic mode)")
 	enhanceCmd.Flags().StringVar(&enhanceMode, "mode", "agentic", "Enhancement mode: agentic (thorough) or single-shot (fast)")
 	enhanceCmd.Flags().StringVar(&enhanceCheckpoint, "checkpoint", "", "Path to save/resume checkpoint (agentic mode)")
+	enhanceCmd.Flags().IntVar(&enhanceWorkers, "workers", 8, "Number of parallel workers for LLM steps (default: 8)")
+	enhanceCmd.Flags().IntVar(&enhanceBackoff, "backoff", 30, "Seconds to wait when rate-limited (default: 30)")
 }
 
 func runEnhance(cmd *cobra.Command, args []string) {
@@ -64,6 +70,18 @@ func runEnhance(cmd *cobra.Command, args []string) {
 		os.Exit(2)
 	}
 
+	// Auto-detect checkpoints from a previous interrupted run
+	if enhanceCheckpoint == "" && ctx != nil {
+		if cpInfo := checkpoint.DetectViaPython(rt.Path, ctx.ScanDir, "enhance"); cpInfo != nil {
+			if checkpoint.PromptResume(cpInfo, "enhance", quiet) {
+				enhanceCheckpoint = cpInfo.Dir
+			} else {
+				// User chose fresh start — remove old checkpoints
+				_ = checkpoint.Clean(cpInfo.Dir)
+			}
+		}
+	}
+
 	pyArgs := []string{"enhance", datasetPath}
 	if enhanceOutput != "" {
 		pyArgs = append(pyArgs, "--output", enhanceOutput)
@@ -80,14 +98,22 @@ func runEnhance(cmd *cobra.Command, args []string) {
 	if enhanceCheckpoint != "" {
 		pyArgs = append(pyArgs, "--checkpoint", enhanceCheckpoint)
 	}
+	if enhanceWorkers != 8 {
+		pyArgs = append(pyArgs, "--workers", fmt.Sprintf("%d", enhanceWorkers))
+	}
+	if enhanceBackoff != 30 {
+		pyArgs = append(pyArgs, "--backoff", fmt.Sprintf("%d", enhanceBackoff))
+	}
 
 	result, err := python.Invoke(rt.Path, pyArgs, "", quiet, requireAPIKey())
 	if err != nil {
 		output.PrintError(err.Error())
 		os.Exit(2)
 	}
 
-	if jsonOutput {
+	if result.Envelope.Status == "interrupted" {
+		os.Exit(130)
+	} else if jsonOutput {
 		output.PrintJSON(result.Envelope)
 	} else if result.Envelope.Status == "success" {
 		if data, ok := result.Envelope.Data.(map[string]any); ok {