Add current local-runtime and inference wire contracts

crates/tinyinference/src/cache/mod.rs

@@ -297,5 +297,32 @@ impl CacheLayoutEvent {
     }
 }
 
+impl CachePolicy {
+    /// Creates a policy with response caching enabled and no expiry.
+    pub fn enabled() -> Self {
+        Self {
+            response_cache_enabled: true,
+            ..Self::default()
+        }
+    }
+
+    /// Returns the configured entry TTL.
+    pub fn ttl(&self) -> Option<std::time::Duration> {
+        self.ttl_ms.map(std::time::Duration::from_millis)
+    }
+
+    /// Sets the entry TTL.
+    pub fn with_ttl(mut self, ttl: std::time::Duration) -> Self {
+        self.ttl_ms = Some(ttl.as_millis() as u64);
+        self
+    }
+
+    /// Sets the cache-key namespace.
+    pub fn with_namespace(mut self, namespace: impl Into<String>) -> Self {
+        self.namespace = Some(namespace.into());
+        self
+    }
+}
+
 #[cfg(test)]
 mod test;

crates/tinyinference/src/cache/types.rs

@@ -130,4 +130,10 @@ pub struct CachePolicy {
     /// When `true`, middleware must preserve the order and content of cacheable
     /// prefix segments. Violations are reported as [`CacheLayoutEvent`]s.
     pub protect_prompt_prefix: bool,
+    /// Entry time-to-live in milliseconds; `None` means no expiry.
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub ttl_ms: Option<u64>,
+    /// Optional cache-key namespace.
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub namespace: Option<String>,
 }

crates/tinyinference/src/embeddings/mod.rs

@@ -294,6 +294,7 @@ mod voyage;
 pub use noop::NoopEmbeddingModel;
 pub use ollama::{
     DEFAULT_OLLAMA_DIMENSIONS, DEFAULT_OLLAMA_MODEL, DEFAULT_OLLAMA_URL, OllamaEmbeddingModel,
+    RECOMMENDED_OLLAMA_CONTEXT_TOKENS,
 };
 pub use openai::OpenAiEmbeddingModel;
 pub use rate_limit::{DEFAULT_REQUESTS_PER_MINUTE, acquire, rate_limit, set_rate_limit};

crates/tinyinference/src/embeddings/ollama.rs

@@ -2,6 +2,10 @@
 
 use async_trait::async_trait;
 use serde::{Deserialize, Serialize};
+use std::sync::{
+    Arc,
+    atomic::{AtomicUsize, Ordering},
+};
 
 use super::EmbeddingModel;
 use crate::{Error, Result};
@@ -12,14 +16,17 @@ pub const DEFAULT_OLLAMA_URL: &str = "http://localhost:11434";
 pub const DEFAULT_OLLAMA_MODEL: &str = "bge-m3";
 /// Default output dimensionality used when zero is requested.
 pub const DEFAULT_OLLAMA_DIMENSIONS: usize = 1024;
+/// Context and batch window recommended for long-document embedding models.
+pub const RECOMMENDED_OLLAMA_CONTEXT_TOKENS: u32 = 8192;
 
 /// Client for Ollama's native `/api/embed` endpoint.
 #[derive(Debug)]
 pub struct OllamaEmbeddingModel {
     client: reqwest::Client,
     base_url: String,
     model: String,
-    dimensions: usize,
+    dimensions: Arc<AtomicUsize>,
+    options: Option<OllamaOptions>,
 }
 
 impl OllamaEmbeddingModel {
@@ -33,14 +40,47 @@ impl OllamaEmbeddingModel {
             client: reqwest::Client::new(),
             base_url: normalize_base_url(base_url)?,
             model: normalize_model(model)?,
-            dimensions: if dimensions == 0 {
+            dimensions: Arc::new(AtomicUsize::new(if dimensions == 0 {
                 DEFAULT_OLLAMA_DIMENSIONS
             } else {
                 dimensions
-            },
+            })),
+            options: None,
         })
     }
 
+    fn try_new_unresolved(base_url: &str, model: &str) -> Result<Self> {
+        Ok(Self {
+            client: reqwest::Client::new(),
+            base_url: normalize_base_url(base_url)?,
+            model: normalize_model(model)?,
+            dimensions: Arc::new(AtomicUsize::new(0)),
+            options: None,
+        })
+    }
+
+    /// Embeds inputs while learning the installed model's vector width.
+    pub async fn embed_discovering_dimensions(
+        base_url: &str,
+        model: &str,
+        client: reqwest::Client,
+        texts: &[String],
+        num_ctx: u32,
+        num_batch: u32,
+    ) -> Result<(usize, Vec<Vec<f32>>)> {
+        if !texts.iter().any(|text| !text.trim().is_empty()) {
+            return Err(Error::Validation(
+                "dynamic embedding dimension discovery requires at least one nonblank input"
+                    .to_string(),
+            ));
+        }
+        let adapter = Self::try_new_unresolved(base_url, model)?
+            .with_client(client)
+            .with_context_options(num_ctx, num_batch);
+        let vectors = adapter.embed(texts).await?;
+        Ok((adapter.dimensions(), vectors))
+    }
+
     /// Creates an Ollama model, panicking for an invalid configuration.
     ///
     /// # Panics
@@ -57,6 +97,15 @@ impl OllamaEmbeddingModel {
         self
     }
 
+    /// Requests an explicit context and batch window from Ollama.
+    pub fn with_context_options(mut self, num_ctx: u32, num_batch: u32) -> Self {
+        self.options = Some(OllamaOptions {
+            num_ctx: num_ctx.max(1),
+            num_batch: num_batch.max(1),
+        });
+        self
+    }
+
     /// Returns the normalized API base URL.
     pub fn base_url(&self) -> &str {
         &self.base_url
@@ -78,6 +127,7 @@ impl OllamaEmbeddingModel {
             .json(&OllamaRequest {
                 model: self.model.clone(),
                 input,
+                options: self.options,
             })
             .send()
             .await
@@ -126,10 +176,24 @@ impl OllamaEmbeddingModel {
     }
 
     fn validate_dimensions(&self, index: usize, vector: &[f32]) -> Result<()> {
-        if vector.len() != self.dimensions {
+        if vector.is_empty() {
+            return Err(Error::Embedding(format!(
+                "ollama embed returned an empty vector at index {index}"
+            )));
+        }
+        let expected = match self.dimensions.compare_exchange(
+            0,
+            vector.len(),
+            Ordering::AcqRel,
+            Ordering::Acquire,
+        ) {
+            Ok(_) => vector.len(),
+            Err(expected) => expected,
+        };
+        if vector.len() != expected {
             return Err(Error::Embedding(format!(
                 "ollama embed dimension mismatch at index {index}: expected {}, got {}",
-                self.dimensions,
+                expected,
                 vector.len()
             )));
         }
@@ -151,6 +215,14 @@ impl Default for OllamaEmbeddingModel {
 struct OllamaRequest {
     model: String,
     input: Vec<String>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    options: Option<OllamaOptions>,
+}
+
+#[derive(Clone, Copy, Debug, Serialize)]
+struct OllamaOptions {
+    num_ctx: u32,
+    num_batch: u32,
 }
 
 #[derive(Deserialize)]
@@ -253,7 +325,7 @@ impl EmbeddingModel for OllamaEmbeddingModel {
     }
 
     fn dimensions(&self) -> usize {
-        self.dimensions
+        self.dimensions.load(Ordering::Acquire)
     }
 
     async fn embed(&self, texts: &[String]) -> Result<Vec<Vec<f32>>> {
@@ -263,13 +335,13 @@ impl EmbeddingModel for OllamaEmbeddingModel {
         let live = texts
             .iter()
             .enumerate()
-            .filter(|(_, text)| !text.trim().is_empty())
-            .map(|(index, text)| (index, text.clone()))
+            .filter_map(|(index, text)| {
+                let text = text.trim();
+                (!text.is_empty()).then(|| (index, text.to_owned()))
+            })
             .collect::<Vec<_>>();
         if live.is_empty() {
-            return Err(Error::Validation(
-                "Ollama embedding batches must not contain blank inputs".into(),
-            ));
+            return Ok(vec![Vec::new(); texts.len()]);
         }
         if live.len() != texts.len() {
             return Err(Error::Validation(
@@ -335,10 +407,10 @@ mod tests {
     }
 
     #[tokio::test]
-    async fn blank_inputs_are_rejected_without_network() {
+    async fn blank_inputs_are_position_safe_without_network() {
         let model = OllamaEmbeddingModel::default();
-        let error = model.embed(&[" ".into(), "\n".into()]).await.unwrap_err();
-        assert!(matches!(error, Error::Validation(_)));
+        let vectors = model.embed(&[" ".into(), "\n".into()]).await.unwrap();
+        assert_eq!(vectors, vec![Vec::<f32>::new(), Vec::new()]);
     }
 
     #[test]

crates/tinyinference/src/message/mod.rs

@@ -129,6 +129,8 @@ impl Message {
         Message::Tool(ToolMessage {
             tool_call_id: tool_call_id.into(),
             content: vec![ContentBlock::Text(content.into())],
+            trusted_verbatim: false,
+            artifact: None,
         })
     }
 
@@ -142,6 +144,14 @@ impl Message {
         }
     }
 
+    /// Returns an out-of-band tool artifact when this is a tool message.
+    pub fn artifact(&self) -> Option<&serde_json::Value> {
+        match self {
+            Message::Tool(message) => message.artifact.as_ref(),
+            _ => None,
+        }
+    }
+
     /// Returns the total number of Unicode scalar values across all text content
     /// blocks, without allocating the concatenated string.
     ///
@@ -162,9 +172,8 @@ impl Message {
             .sum()
     }
 
-    /// Approximate character weight of the message across *all* content blocks
-    /// (text, JSON, images, reasoning, provider extensions), for token
-    /// estimation and context-window gating.
+    /// Approximate character weight of provider-visible content and structural
+    /// tool-call payloads, for token estimation and context-window gating.
     ///
     /// Distinct from [`char_len`](Self::char_len), which counts only visible
     /// text: a transcript dominated by images, large tool-result JSON, or model
@@ -178,11 +187,31 @@ impl Message {
             Message::Assistant(m) => &m.content,
             Message::Tool(m) => &m.content,
         };
-        content
+        let content_weight: usize = content
             .iter()
             .map(ContentBlock::estimated_char_weight)
-            .sum()
+            .sum();
+        let structural_weight = match self {
+            Message::Assistant(message) => tool_calls_char_weight(&message.tool_calls),
+            Message::Tool(message) => message.tool_call_id.chars().count(),
+            _ => 0,
+        };
+        content_weight + structural_weight
+    }
+}
+
+fn tool_calls_char_weight(tool_calls: &[crate::tool::ToolCall]) -> usize {
+    if tool_calls.is_empty() {
+        return 0;
     }
+    serde_json::to_string(tool_calls)
+        .map(|rendered| rendered.chars().count())
+        .unwrap_or_else(|_| {
+            tool_calls
+                .iter()
+                .map(|call| call.name.chars().count() + call.arguments.to_string().chars().count())
+                .sum()
+        })
 }
 
 #[cfg(test)]

crates/tinyinference/src/message/types.rs

@@ -99,6 +99,16 @@ pub struct ToolMessage {
     pub tool_call_id: String,
     /// Ordered content blocks.
     pub content: Vec<ContentBlock>,
+    /// Whether a consuming runtime must preserve the content byte-for-byte.
+    #[serde(default, skip_serializing_if = "is_false")]
+    pub trusted_verbatim: bool,
+    /// Host-side structured payload that is never sent to the provider.
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub artifact: Option<Value>,
+}
+
+fn is_false(value: &bool) -> bool {
+    !*value
 }
 
 /// A structured conversation message.

crates/tinyinference/src/model/mod.rs

@@ -47,6 +47,7 @@ enum ContextPatternMatch {
 /// substrings such as `gpt-4.1` and `gpt-4-turbo` must stay before broader
 /// patterns such as `gpt-4` that would otherwise shadow them.
 const MODEL_CONTEXT_PATTERNS: &[(&str, ContextPatternMatch, u64)] = &[
+    ("gpt-5", ContextPatternMatch::Substring, 400_000),
     ("claude-haiku-4.5", ContextPatternMatch::Substring, 200_000),
     ("claude-haiku-4", ContextPatternMatch::Substring, 200_000),
     ("claude-haiku", ContextPatternMatch::Substring, 200_000),
@@ -161,6 +162,7 @@ impl ModelProfile {
             && (!set.native_structured_output || self.native_structured_output)
             && (!set.json_schema || self.json_schema)
             && (!set.reasoning || self.reasoning)
+            && (!set.reasoning_effort || self.reasoning_effort)
             && (!set.image_in || self.modalities.image_in)
             && (!set.image_out || self.modalities.image_out)
             && (!set.audio_in || self.modalities.audio_in)
@@ -215,6 +217,7 @@ impl ModelProfile {
             native_structured_output: true,
             json_schema: true,
             reasoning: true,
+            reasoning_effort: true,
             ..Self::default()
         }
     }
@@ -356,6 +359,17 @@ impl ModelRequest {
         self
     }
 
+    /// Sets provider-neutral reasoning configuration.
+    pub fn with_reasoning(mut self, reasoning: ReasoningConfig) -> Self {
+        self.reasoning = Some(reasoning);
+        self
+    }
+
+    /// Sets only the requested reasoning effort.
+    pub fn with_reasoning_effort(self, effort: ReasoningEffort) -> Self {
+        self.with_reasoning(ReasoningConfig::effort(effort))
+    }
+
     /// Returns the ids of cacheable segments in declaration order, describing
     /// the stable prompt prefix middleware should preserve.
     pub fn cacheable_prefix_ids(&self) -> Vec<String> {
@@ -381,6 +395,8 @@ impl ModelResponse {
             finish_reason: None,
             raw: None,
             resolved_model: None,
+            continue_turn: None,
+            served_from_cache: false,
         }
     }
 
@@ -609,6 +625,8 @@ impl StreamAccumulator {
             finish_reason: None,
             raw: None,
             resolved_model: None,
+            continue_turn: None,
+            served_from_cache: false,
         })
     }
 }