feat(describe): structured output for the skip decision (T-081, keystone)

config.example.yaml

@@ -16,6 +16,10 @@ api:
   # and has a vision-capable model. REQUIRED — edit before first run.
   base_url: "https://your-llm-endpoint.example/v1"
   model: "your-vision-model-name"
+  # Sampling temperature for every model call. 0 (the default) gives reproducible
+  # output, which is what makes "did this change help?" answerable. Raise only if
+  # you deliberately want variation.
+  temperature: 0
   # OPTIONAL cost estimation. Token usage is always reported; a monetary estimate
   # is only added when BOTH prices below are set (per single token — the unit most
   # OpenAI-compatible /v1/models endpoints use). Provider-neutral: look up your
@@ -90,7 +94,7 @@ diagrams:
 
     Strukturera svaret så här:
     1. Vad diagrammet visar: titel/ämne och vad x- respektive y-axeln mäter (inklusive enheter).
-    2. Vilka serier/data: nämn alla serier som visas (linjer, staplar, etc.) och hur de skiljs åt visuellt.
+    2. Vilka serier/data: nämn alla serier som visas (linjer, staplar, etc.) och hur de skiljs åt visuellt. Beskriv varje serie med hur den utvecklas, helst med konkreta siffror.
     3. De viktigaste observationerna: trender, toppar, bottnar, brytpunkter, divergenser. Var konkret med siffror och årtal.
     4. Eventuella anmärkningar och källor om de finns med i bilden.
 

src/figmark/config.py

@@ -29,6 +29,9 @@ class ApiConfig:
     input_token_price: float | None = None
     output_token_price: float | None = None
     currency: str | None = None
+    # Sampling temperature for every model call. Defaults to 0 for reproducible
+    # output — the prerequisite for measuring whether a change helped (T-082).
+    temperature: float = 0.0
 
 
 @dataclass
@@ -214,6 +217,7 @@ def load_config(config_path: str | Path = "config.yaml") -> Config:
         input_token_price=in_price,
         output_token_price=out_price,
         currency=(str(api_raw["currency"]).strip() if api_raw.get("currency") else None),
+        temperature=float(api_raw.get("temperature", 0.0) or 0.0),
     )
 
     input_raw = raw.get("input") or {}

src/figmark/describe.py

@@ -17,8 +17,9 @@
 import time
 from pathlib import Path
 
-from openai import APIError, APITimeoutError, OpenAI, RateLimitError
+from openai import APIError, APITimeoutError, BadRequestError, OpenAI, RateLimitError
 from PIL import Image
+from pydantic import BaseModel, ValidationError
 
 from .config import Config
 from .context import ContextText
@@ -66,6 +67,117 @@ def is_skip(text: str | None) -> bool:
     return text is not None and text.strip().upper().startswith(SKIP_MARKER)
 
 
+# --- Structured describe (T-081) -------------------------------------------
+# The skip/keep decision comes from a validated boolean, not a fragile "[SKIP]"
+# string the model must format perfectly. On an endpoint without json_schema
+# support we fall back to the legacy text prompt + is_skip() (air-gap-safe).
+
+_KIND_ENUM = ["chart", "table", "diagram", "photo", "illustration", "decoration", "text", "other"]
+
+
+class FigureResult(BaseModel):
+    is_figure: bool
+    kind: str
+    description: str
+
+
+_FIGURE_SCHEMA = {
+    "type": "json_schema",
+    "json_schema": {
+        "name": "figure_result",
+        "strict": True,
+        "schema": {
+            "type": "object",
+            "properties": {
+                "is_figure": {
+                    "type": "boolean",
+                    "description": (
+                        "false if the image is purely decorative — a logo, icon, divider, "
+                        "background or border that carries no information a reader needs; "
+                        "true if it conveys information worth describing"
+                    ),
+                },
+                "kind": {"type": "string", "enum": _KIND_ENUM, "description": "what the image is"},
+                "description": {
+                    "type": "string",
+                    "description": "the description; empty string when is_figure is false",
+                },
+            },
+            "required": ["is_figure", "kind", "description"],
+            "additionalProperties": False,
+        },
+    },
+}
+
+# Endpoints that rejected response_format=json_schema — probed once, then we skip
+# the structured attempt and go straight to the text path.
+_structured_unsupported: set[str] = set()
+
+
+def _image_messages(text: str, data_uri: str) -> list[dict]:
+    return [
+        {
+            "role": "user",
+            "content": [
+                {"type": "text", "text": text},
+                {"type": "image_url", "image_url": {"url": data_uri}},
+            ],
+        }
+    ]
+
+
+def call_vision(
+    client: OpenAI,
+    cfg: Config,
+    structured_text: str,
+    fallback_text: str,
+    data_uri: str,
+    max_tokens: int,
+) -> tuple[str, bool]:
+    """One model round. Returns (text, truncated); text is the description or SKIP_MARKER.
+
+    Structured output first (a validated ``is_figure`` boolean); on an endpoint
+    that doesn't support ``response_format=json_schema`` we fall back to the
+    legacy text prompt (T-081). Transient errors propagate to the caller's retry
+    loop; only the "structured unsupported" 400 is handled here.
+    """
+    key = f"{cfg.api.base_url}|{cfg.api.model}"
+    if key not in _structured_unsupported:
+        try:
+            resp = client.chat.completions.create(  # type: ignore[call-overload]
+                model=cfg.api.model,
+                temperature=cfg.api.temperature,
+                max_tokens=max_tokens,
+                response_format=_FIGURE_SCHEMA,
+                messages=_image_messages(structured_text, data_uri),
+            )
+        except BadRequestError:
+            _structured_unsupported.add(key)  # endpoint lacks json_schema support
+        else:
+            choice = resp.choices[0]
+            content = (choice.message.content or "").strip()
+            truncated = getattr(choice, "finish_reason", None) == "length"
+            try:
+                fr = FigureResult.model_validate_json(content)
+            except (ValidationError, ValueError):
+                pass  # malformed/cut JSON this once — fall through to the text path
+            else:
+                if not fr.is_figure:
+                    return SKIP_MARKER, False
+                return fr.description.strip(), truncated
+
+    resp = client.chat.completions.create(  # type: ignore[call-overload]
+        model=cfg.api.model,
+        temperature=cfg.api.temperature,
+        max_tokens=max_tokens,
+        messages=_image_messages(fallback_text, data_uri),  # type: ignore[arg-type]
+    )
+    choice = resp.choices[0]
+    text = (choice.message.content or "").strip()
+    truncated = getattr(choice, "finish_reason", None) == "length"
+    return text, truncated
+
+
 def cache_fingerprint(*parts: object) -> str:
     """Short, stable hash of the inputs that determine a description's content.
 
@@ -185,43 +297,39 @@ def describe_image(
             return cached
 
     data_uri = _image_to_data_uri(image_path)
-    user_text = compose_prompt(
+    lang = language if language is not None else cfg.language.output
+    # Structured prompt drops the [SKIP] instruction — the schema's is_figure
+    # carries the skip decision (T-081); the fallback prompt keeps it.
+    structured_text = compose_prompt(
+        cfg.description.prompt,
+        doc_summary=doc_summary,
+        context=context,
+        significance=False,
+        language=lang,
+    )
+    fallback_text = compose_prompt(
         cfg.description.prompt,
         doc_summary=doc_summary,
         context=context,
         significance=cfg.significance.enabled,
-        language=language if language is not None else cfg.language.output,
+        language=lang,
     )
 
     last_error: Exception | None = None
     for attempt in range(1, MAX_RETRIES + 1):
         try:
-            response = client.chat.completions.create(
-                model=cfg.api.model,
-                max_tokens=MAX_TOKENS,
-                messages=[
-                    {
-                        "role": "user",
-                        "content": [
-                            {"type": "text", "text": user_text},
-                            {"type": "image_url", "image_url": {"url": data_uri}},
-                        ],
-                    }
-                ],
+            text, truncated = call_vision(
+                client, cfg, structured_text, fallback_text, data_uri, MAX_TOKENS
             )
-            choice = response.choices[0]
-            text = (choice.message.content or "").strip()
             if not text:
-                # An empty completion is a hard error, deliberately NOT retried:
-                # with FAIL_FAST a retry just repeats the same failure, and an empty
-                # body almost always means the model can't take image input or
-                # rejected the prompt — not a transient blip. (T-033)
+                # A figure with no description is a hard error, deliberately NOT
+                # retried: an empty body almost always means the model can't take
+                # image input or rejected the prompt — not a transient blip (T-033).
                 raise RuntimeError(
                     f"The API returned empty content for {image_path.name} "
                     f"(model={cfg.api.model}). This usually means the model does "
                     f"not support image input or the prompt was rejected."
                 )
-            truncated = getattr(choice, "finish_reason", None) == "length"
             if truncated:
                 # Truncated at the token cap — the description is likely cut
                 # mid-sentence. Warn loudly (do not silently cache a partial as if

src/figmark/diagrams.py

@@ -466,6 +466,7 @@ def describe_diagram(
 
     response = client.chat.completions.create(
         model=cfg.api.model,
+        temperature=cfg.api.temperature,
         max_tokens=MAX_TOKENS,
         messages=[
             {

src/figmark/ocr.py

@@ -192,6 +192,7 @@ def ocr_page_with_vision(
     try:
         response = client.chat.completions.create(
             model=cfg.api.model,
+            temperature=cfg.api.temperature,
             max_tokens=OCR_MAX_TOKENS,
             messages=[
                 {

src/figmark/summarize.py

@@ -48,6 +48,7 @@ def detect_language(client, pages, cfg: Config, cache_path: Path) -> str:
 
     response = client.chat.completions.create(
         model=cfg.api.model,
+        temperature=cfg.api.temperature,
         max_tokens=LANG_DETECT_MAX_TOKENS,
         messages=[{"role": "user", "content": f"{LANG_DETECT_PROMPT}\n\nText:\n{sample}"}],
     )
@@ -115,6 +116,7 @@ def summarize_document(client, pages, cfg: Config, cache_path: Path, language: s
     )
     response = client.chat.completions.create(
         model=cfg.api.model,
+        temperature=cfg.api.temperature,
         max_tokens=SUMMARY_MAX_TOKENS,
         messages=[{"role": "user", "content": user_text}],
     )

tests/fakes.py

@@ -9,6 +9,7 @@
 
 from __future__ import annotations
 
+import json
 from pathlib import Path
 from types import SimpleNamespace
 
@@ -73,6 +74,18 @@ def _create(self, model, max_tokens, messages, **kwargs):
             return make_response(SUMMARY_REPLY)
         text = next(part["text"] for part in content if part["type"] == "text")
         self.describe_prompts.append(text)
+        if "response_format" in kwargs:
+            # Emulate structured output (T-081): is_figure=false for a skip reply,
+            # description carries the canned text otherwise.
+            skip = self.image_reply.strip().upper().startswith("[SKIP]")
+            payload = json.dumps(
+                {
+                    "is_figure": not skip,
+                    "kind": "chart",
+                    "description": "" if skip else self.image_reply,
+                }
+            )
+            return make_response(payload, finish_reason=self.finish_reason)
         return make_response(self.image_reply, finish_reason=self.finish_reason)
 
 

tests/test_structured_describe.py

@@ -0,0 +1,50 @@
+"""T-081: the skip/keep decision comes from a validated is_figure boolean, and
+the text path is a working fallback for endpoints without json_schema support."""
+
+from __future__ import annotations
+
+from types import SimpleNamespace
+
+import pytest
+
+from figmark.describe import SKIP_MARKER, _structured_unsupported, call_vision
+
+from .fakes import FakeClient
+
+
+def _cfg(base_url: str) -> SimpleNamespace:
+    return SimpleNamespace(api=SimpleNamespace(model="m", base_url=base_url, temperature=0.0))
+
+
+@pytest.fixture(autouse=True)
+def _clear_unsupported():
+    _structured_unsupported.clear()
+    yield
+    _structured_unsupported.clear()
+
+
+def test_structured_skip_uses_is_figure_not_the_string():
+    """A decorative image -> the mock returns is_figure=false -> clean skip,
+    via the structured prompt (no fragile "[SKIP]" string parsing)."""
+    client = FakeClient("[SKIP]")
+    text, truncated = call_vision(client, _cfg("http://a"), "STRUCTURED", "FALLBACK", "data:x", 100)
+    assert text == SKIP_MARKER
+    assert not truncated
+    assert client.describe_prompts[-1] == "STRUCTURED"
+
+
+def test_structured_describe_returns_description():
+    client = FakeClient("A bar chart of exports.")
+    text, _ = call_vision(client, _cfg("http://b"), "STRUCTURED", "FALLBACK", "data:x", 100)
+    assert text == "A bar chart of exports."
+    assert client.describe_prompts[-1] == "STRUCTURED"
+
+
+def test_falls_back_to_text_when_endpoint_lacks_structured_support():
+    """An endpoint known to reject json_schema -> the legacy text prompt runs,
+    and its "[SKIP]" reply is still honoured via is_skip() downstream."""
+    _structured_unsupported.add("http://c|m")
+    client = FakeClient("A line chart.")
+    text, _ = call_vision(client, _cfg("http://c"), "STRUCTURED", "FALLBACK", "data:x", 100)
+    assert text == "A line chart."
+    assert client.describe_prompts[-1] == "FALLBACK"  # the text path, not structured

tests/test_summarize.py

@@ -59,7 +59,7 @@ def _stub_client(text: str, finish_reason: str = "stop"):
 
 
 _CFG = SimpleNamespace(
-    api=SimpleNamespace(model="test-model"),
+    api=SimpleNamespace(model="test-model", temperature=0.0),
     document_summary=SimpleNamespace(sample_words=50, prompt="Summarise."),
 )