From 940c2983da7f6af9ec95367e4f594777627e3f0e Mon Sep 17 00:00:00 2001 From: Joel Stenberg Date: Thu, 9 Jul 2026 13:09:58 +0200 Subject: [PATCH 1/2] feat(config): temperature=0 for reproducible output on every model call (Phase 0, T-082) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds api.temperature (default 0.0) and passes it to all five completion call sites (describe, diagram, OCR, and both summarize calls). Reproducibility is the prerequisite for the quality track: without it, the same document gives different output each run, so "did this change help?" is unanswerable. Also carries Joel's config.example.yaml prompt refinement (item 2: describe each series' development, ideally with concrete numbers) — an unrelated one-liner that was in the working tree; kept so it isn't lost. Co-Authored-By: Claude Opus 4.8 --- config.example.yaml | 6 +++++- src/figmark/config.py | 4 ++++ src/figmark/describe.py | 1 + src/figmark/diagrams.py | 1 + src/figmark/ocr.py | 1 + src/figmark/summarize.py | 2 ++ tests/test_summarize.py | 2 +- 7 files changed, 15 insertions(+), 2 deletions(-) diff --git a/config.example.yaml b/config.example.yaml index 2de2082..90310b8 100644 --- a/config.example.yaml +++ b/config.example.yaml @@ -16,6 +16,10 @@ api: # and has a vision-capable model. REQUIRED — edit before first run. base_url: "https://your-llm-endpoint.example/v1" model: "your-vision-model-name" + # Sampling temperature for every model call. 0 (the default) gives reproducible + # output, which is what makes "did this change help?" answerable. Raise only if + # you deliberately want variation. + temperature: 0 # OPTIONAL cost estimation. Token usage is always reported; a monetary estimate # is only added when BOTH prices below are set (per single token — the unit most # OpenAI-compatible /v1/models endpoints use). Provider-neutral: look up your @@ -90,7 +94,7 @@ diagrams: Strukturera svaret så här: 1. Vad diagrammet visar: titel/ämne och vad x- respektive y-axeln mäter (inklusive enheter). - 2. Vilka serier/data: nämn alla serier som visas (linjer, staplar, etc.) och hur de skiljs åt visuellt. + 2. Vilka serier/data: nämn alla serier som visas (linjer, staplar, etc.) och hur de skiljs åt visuellt. Beskriv varje serie med hur den utvecklas, helst med konkreta siffror. 3. De viktigaste observationerna: trender, toppar, bottnar, brytpunkter, divergenser. Var konkret med siffror och årtal. 4. Eventuella anmärkningar och källor om de finns med i bilden. diff --git a/src/figmark/config.py b/src/figmark/config.py index 9c506d2..bf5b813 100644 --- a/src/figmark/config.py +++ b/src/figmark/config.py @@ -29,6 +29,9 @@ class ApiConfig: input_token_price: float | None = None output_token_price: float | None = None currency: str | None = None + # Sampling temperature for every model call. Defaults to 0 for reproducible + # output — the prerequisite for measuring whether a change helped (T-082). + temperature: float = 0.0 @dataclass @@ -214,6 +217,7 @@ def load_config(config_path: str | Path = "config.yaml") -> Config: input_token_price=in_price, output_token_price=out_price, currency=(str(api_raw["currency"]).strip() if api_raw.get("currency") else None), + temperature=float(api_raw.get("temperature", 0.0) or 0.0), ) input_raw = raw.get("input") or {} diff --git a/src/figmark/describe.py b/src/figmark/describe.py index f7ff360..d3a9978 100644 --- a/src/figmark/describe.py +++ b/src/figmark/describe.py @@ -198,6 +198,7 @@ def describe_image( try: response = client.chat.completions.create( model=cfg.api.model, + temperature=cfg.api.temperature, max_tokens=MAX_TOKENS, messages=[ { diff --git a/src/figmark/diagrams.py b/src/figmark/diagrams.py index 341df5e..48a1b9e 100644 --- a/src/figmark/diagrams.py +++ b/src/figmark/diagrams.py @@ -466,6 +466,7 @@ def describe_diagram( response = client.chat.completions.create( model=cfg.api.model, + temperature=cfg.api.temperature, max_tokens=MAX_TOKENS, messages=[ { diff --git a/src/figmark/ocr.py b/src/figmark/ocr.py index 159a5cf..978dac3 100644 --- a/src/figmark/ocr.py +++ b/src/figmark/ocr.py @@ -192,6 +192,7 @@ def ocr_page_with_vision( try: response = client.chat.completions.create( model=cfg.api.model, + temperature=cfg.api.temperature, max_tokens=OCR_MAX_TOKENS, messages=[ { diff --git a/src/figmark/summarize.py b/src/figmark/summarize.py index 92090d2..6e57724 100644 --- a/src/figmark/summarize.py +++ b/src/figmark/summarize.py @@ -48,6 +48,7 @@ def detect_language(client, pages, cfg: Config, cache_path: Path) -> str: response = client.chat.completions.create( model=cfg.api.model, + temperature=cfg.api.temperature, max_tokens=LANG_DETECT_MAX_TOKENS, messages=[{"role": "user", "content": f"{LANG_DETECT_PROMPT}\n\nText:\n{sample}"}], ) @@ -115,6 +116,7 @@ def summarize_document(client, pages, cfg: Config, cache_path: Path, language: s ) response = client.chat.completions.create( model=cfg.api.model, + temperature=cfg.api.temperature, max_tokens=SUMMARY_MAX_TOKENS, messages=[{"role": "user", "content": user_text}], ) diff --git a/tests/test_summarize.py b/tests/test_summarize.py index 162527f..071971f 100644 --- a/tests/test_summarize.py +++ b/tests/test_summarize.py @@ -59,7 +59,7 @@ def _stub_client(text: str, finish_reason: str = "stop"): _CFG = SimpleNamespace( - api=SimpleNamespace(model="test-model"), + api=SimpleNamespace(model="test-model", temperature=0.0), document_summary=SimpleNamespace(sample_words=50, prompt="Summarise."), ) From 108a65bc65760eb137fe8379eb6aaf9013d6ebd5 Mon Sep 17 00:00:00 2001 From: Joel Stenberg Date: Thu, 9 Jul 2026 14:30:09 +0200 Subject: [PATCH 2/2] =?UTF-8?q?wip(T-081):=20structured=20describe=20?= =?UTF-8?q?=E2=80=94=20FigureResult=20+=20call=5Fvision=20+=20fakes=20[nee?= =?UTF-8?q?ds=20Phase=200]?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/figmark/describe.py | 152 ++++++++++++++++++++++++++++++++++------ tests/fakes.py | 13 ++++ 2 files changed, 143 insertions(+), 22 deletions(-) diff --git a/src/figmark/describe.py b/src/figmark/describe.py index f7ff360..14ed017 100644 --- a/src/figmark/describe.py +++ b/src/figmark/describe.py @@ -17,8 +17,9 @@ import time from pathlib import Path -from openai import APIError, APITimeoutError, OpenAI, RateLimitError +from openai import APIError, APITimeoutError, BadRequestError, OpenAI, RateLimitError from PIL import Image +from pydantic import BaseModel, ValidationError from .config import Config from .context import ContextText @@ -66,6 +67,117 @@ def is_skip(text: str | None) -> bool: return text is not None and text.strip().upper().startswith(SKIP_MARKER) +# --- Structured describe (T-081) ------------------------------------------- +# The skip/keep decision comes from a validated boolean, not a fragile "[SKIP]" +# string the model must format perfectly. On an endpoint without json_schema +# support we fall back to the legacy text prompt + is_skip() (air-gap-safe). + +_KIND_ENUM = ["chart", "table", "diagram", "photo", "illustration", "decoration", "text", "other"] + + +class FigureResult(BaseModel): + is_figure: bool + kind: str + description: str + + +_FIGURE_SCHEMA = { + "type": "json_schema", + "json_schema": { + "name": "figure_result", + "strict": True, + "schema": { + "type": "object", + "properties": { + "is_figure": { + "type": "boolean", + "description": ( + "false if the image is purely decorative — a logo, icon, divider, " + "background or border that carries no information a reader needs; " + "true if it conveys information worth describing" + ), + }, + "kind": {"type": "string", "enum": _KIND_ENUM, "description": "what the image is"}, + "description": { + "type": "string", + "description": "the description, following the task; empty string when is_figure is false", + }, + }, + "required": ["is_figure", "kind", "description"], + "additionalProperties": False, + }, + }, +} + +# Endpoints that rejected response_format=json_schema — probed once, then we skip +# the structured attempt and go straight to the text path. +_structured_unsupported: set[str] = set() + + +def _image_messages(text: str, data_uri: str) -> list[dict]: + return [ + { + "role": "user", + "content": [ + {"type": "text", "text": text}, + {"type": "image_url", "image_url": {"url": data_uri}}, + ], + } + ] + + +def call_vision( + client: OpenAI, + cfg: Config, + structured_text: str, + fallback_text: str, + data_uri: str, + max_tokens: int, +) -> tuple[str, bool]: + """One model round. Returns (text, truncated); text is the description or SKIP_MARKER. + + Structured output first (a validated ``is_figure`` boolean); on an endpoint + that doesn't support ``response_format=json_schema`` we fall back to the + legacy text prompt (T-081). Transient errors propagate to the caller's retry + loop; only the "structured unsupported" 400 is handled here. + """ + key = f"{cfg.api.base_url}|{cfg.api.model}" + if key not in _structured_unsupported: + try: + resp = client.chat.completions.create( + model=cfg.api.model, + temperature=cfg.api.temperature, + max_tokens=max_tokens, + response_format=_FIGURE_SCHEMA, + messages=_image_messages(structured_text, data_uri), + ) + except BadRequestError: + _structured_unsupported.add(key) # endpoint lacks json_schema support + else: + choice = resp.choices[0] + content = (choice.message.content or "").strip() + truncated = getattr(choice, "finish_reason", None) == "length" + try: + fr = FigureResult.model_validate_json(content) + except (ValidationError, ValueError): + pass # malformed/cut JSON this once — fall through to the text path + else: + if not fr.is_figure: + return SKIP_MARKER, False + return fr.description.strip(), truncated + + resp = client.chat.completions.create( + model=cfg.api.model, + temperature=cfg.api.temperature, + max_tokens=max_tokens, + messages=_image_messages(fallback_text, data_uri), + ) + choice = resp.choices[0] + text = (choice.message.content or "").strip() + truncated = getattr(choice, "finish_reason", None) == "length" + return text, truncated + + def cache_fingerprint(*parts: object) -> str: """Short, stable hash of the inputs that determine a description's content. @@ -185,43 +297,39 @@ def describe_image( return cached data_uri = _image_to_data_uri(image_path) - user_text = compose_prompt( + lang = language if language is not None else cfg.language.output + # Structured prompt drops the [SKIP] instruction — the schema's is_figure + # carries the skip decision (T-081); the fallback prompt keeps it. + structured_text = compose_prompt( + cfg.description.prompt, + doc_summary=doc_summary, + context=context, + significance=False, + language=lang, + ) + fallback_text = compose_prompt( cfg.description.prompt, doc_summary=doc_summary, context=context, significance=cfg.significance.enabled, - language=language if language is not None else cfg.language.output, + language=lang, ) last_error: Exception | None = None for attempt in range(1, MAX_RETRIES + 1): try: - response = client.chat.completions.create( - model=cfg.api.model, - max_tokens=MAX_TOKENS, - messages=[ - { - "role": "user", - "content": [ - {"type": "text", "text": user_text}, - {"type": "image_url", "image_url": {"url": data_uri}}, - ], - } - ], + text, truncated = call_vision( + client, cfg, structured_text, fallback_text, data_uri, MAX_TOKENS ) - choice = response.choices[0] - text = (choice.message.content or "").strip() if not text: - # An empty completion is a hard error, deliberately NOT retried: - # with FAIL_FAST a retry just repeats the same failure, and an empty - # body almost always means the model can't take image input or - # rejected the prompt — not a transient blip. (T-033) + # A figure with no description is a hard error, deliberately NOT + # retried: an empty body almost always means the model can't take + # image input or rejected the prompt — not a transient blip (T-033). raise RuntimeError( f"The API returned empty content for {image_path.name} " f"(model={cfg.api.model}). This usually means the model does " f"not support image input or the prompt was rejected." ) - truncated = getattr(choice, "finish_reason", None) == "length" if truncated: # Truncated at the token cap — the description is likely cut # mid-sentence. Warn loudly (do not silently cache a partial as if diff --git a/tests/fakes.py b/tests/fakes.py index 24a5ddb..ed5fe14 100644 --- a/tests/fakes.py +++ b/tests/fakes.py @@ -9,6 +9,7 @@ from __future__ import annotations +import json from pathlib import Path from types import SimpleNamespace @@ -73,6 +74,18 @@ def _create(self, model, max_tokens, messages, **kwargs): return make_response(SUMMARY_REPLY) text = next(part["text"] for part in content if part["type"] == "text") self.describe_prompts.append(text) + if "response_format" in kwargs: + # Emulate structured output (T-081): is_figure=false for a skip reply, + # description carries the canned text otherwise. + skip = self.image_reply.strip().upper().startswith("[SKIP]") + payload = json.dumps( + { + "is_figure": not skip, + "kind": "chart", + "description": "" if skip else self.image_reply, + } + ) + return make_response(payload, finish_reason=self.finish_reason) return make_response(self.image_reply, finish_reason=self.finish_reason)