From e95ecf879e085f8ecbc7e605ebf4866f64d34e5b Mon Sep 17 00:00:00 2001 From: inkpad Date: Tue, 17 Feb 2026 03:18:56 +0000 Subject: [PATCH 01/14] Add uncertainty/certainty intrinsic support Wire up the uncertainty intrinsic from ibm-granite/granite-lib-core-r1.0 with a high-level check_certainty() API. The intrinsic evaluates model confidence in its response given a user question and assistant answer. - Add check_certainty(context, backend) in core.py - Extract shared call_intrinsic() helper into _util.py - Update catalog to point uncertainty at granite-lib-core-r1.0 - Add test, example, and README entry Co-Authored-By: ink-pad --- docs/examples/intrinsics/README.md | 4 + docs/examples/intrinsics/uncertainty.py | 27 +++++++ mellea/backends/adapters/catalog.py | 3 +- mellea/stdlib/components/intrinsic/_util.py | 57 ++++++++++++++ mellea/stdlib/components/intrinsic/core.py | 21 +++++ mellea/stdlib/components/intrinsic/rag.py | 55 +------------ test/stdlib/components/intrinsic/test_core.py | 77 +++++++++++++++++++ .../testdata/input_json/uncertainty.json | 12 +++ 8 files changed, 202 insertions(+), 54 deletions(-) create mode 100644 docs/examples/intrinsics/uncertainty.py create mode 100644 mellea/stdlib/components/intrinsic/_util.py create mode 100644 mellea/stdlib/components/intrinsic/core.py create mode 100644 test/stdlib/components/intrinsic/test_core.py create mode 100644 test/stdlib/components/intrinsic/testdata/input_json/uncertainty.json diff --git a/docs/examples/intrinsics/README.md b/docs/examples/intrinsics/README.md index 5df006912..97c8dabf5 100644 --- a/docs/examples/intrinsics/README.md +++ b/docs/examples/intrinsics/README.md @@ -31,6 +31,9 @@ Detects when model outputs contain hallucinated information. ### query_rewrite.py Rewrites queries for better retrieval or understanding. +### uncertainty.py +Estimates the model's certainty about answering a question given retrieved documents. + ## Concepts Demonstrated - **Intrinsic Functions**: Specialized model capabilities beyond text generation @@ -71,6 +74,7 @@ out, new_ctx = mfuncs.act( - **context_relevance**: Assess context-query relevance - **hallucination_detection**: Detect hallucinated content - **query_rewrite**: Improve query formulation +- **uncertainty**: Estimate certainty about answering a question ## Related Documentation diff --git a/docs/examples/intrinsics/uncertainty.py b/docs/examples/intrinsics/uncertainty.py new file mode 100644 index 000000000..b0ade6e88 --- /dev/null +++ b/docs/examples/intrinsics/uncertainty.py @@ -0,0 +1,27 @@ +# pytest: huggingface, requires_heavy_ram, llm + +"""Example usage of the uncertainty/certainty intrinsic. + +Evaluates how certain the model is about its response to a user question. +The context should contain a user question followed by an assistant answer. + +To run this script from the root of the Mellea source tree, use the command: +``` +uv run python docs/examples/intrinsics/uncertainty.py +``` +""" + +from mellea.backends.huggingface import LocalHFBackend +from mellea.stdlib.components import Message +from mellea.stdlib.components.intrinsic import core +from mellea.stdlib.context import ChatContext + +backend = LocalHFBackend(model_id="ibm-granite/granite-4.0-micro") +context = ( + ChatContext() + .add(Message("user", "What is the square root of 4?")) + .add(Message("assistant", "The square root of 4 is 2.")) +) + +result = core.check_certainty(context, backend) +print(f"Certainty score: {result}") diff --git a/mellea/backends/adapters/catalog.py b/mellea/backends/adapters/catalog.py index 99d773ca4..28ad0e929 100644 --- a/mellea/backends/adapters/catalog.py +++ b/mellea/backends/adapters/catalog.py @@ -40,6 +40,7 @@ class IntriniscsCatalogEntry(pydantic.BaseModel): _RAG_REPO = "ibm-granite/granite-lib-rag-r1.0" _CORE_REPO = "ibm-granite/rag-intrinsics-lib" +_CORE_R1_REPO = "ibm-granite/granite-lib-core-r1.0" _INTRINSICS_CATALOG_ENTRIES = [ @@ -47,7 +48,7 @@ class IntriniscsCatalogEntry(pydantic.BaseModel): # Core Intrinsics ############################################ IntriniscsCatalogEntry(name="requirement_check", repo_id=_CORE_REPO), - IntriniscsCatalogEntry(name="uncertainty", repo_id=_CORE_REPO), + IntriniscsCatalogEntry(name="uncertainty", repo_id=_CORE_R1_REPO), ############################################ # RAG Intrinsics ############################################ diff --git a/mellea/stdlib/components/intrinsic/_util.py b/mellea/stdlib/components/intrinsic/_util.py new file mode 100644 index 000000000..3a60417df --- /dev/null +++ b/mellea/stdlib/components/intrinsic/_util.py @@ -0,0 +1,57 @@ +"""Shared utilities for intrinsic convenience wrappers.""" + +import json + +from ....backends import ModelOption +from ....backends.adapters import AdapterMixin, AdapterType, GraniteCommonAdapter +from ....stdlib import functional as mfuncs +from ...context import ChatContext +from .intrinsic import Intrinsic + + +def call_intrinsic( + intrinsic_name: str, + context: ChatContext, + backend: AdapterMixin, + /, + kwargs: dict | None = None, +): + """Shared code for invoking intrinsics. + + :returns: Result of the call in JSON format. + """ + # Adapter needs to be present in the backend before it can be invoked. + # We must create the Adapter object in order to determine whether we need to create + # the Adapter object. + base_model_name = backend.base_model_name + if base_model_name is None: + raise ValueError("Backend has no model ID") + adapter = GraniteCommonAdapter( + intrinsic_name, adapter_type=AdapterType.LORA, base_model_name=base_model_name + ) + if adapter.qualified_name not in backend.list_adapters(): + backend.add_adapter(adapter) + + # Create the AST node for the action we wish to perform. + intrinsic = Intrinsic(intrinsic_name, intrinsic_kwargs=kwargs) + + # Execute the AST node. + model_output_thunk, _ = mfuncs.act( + intrinsic, + context, + backend, + model_options={ModelOption.TEMPERATURE: 0.0}, + # No rejection sampling, please + strategy=None, + ) + + # act() can return a future. Don't know how to handle one from non-async code. + assert model_output_thunk.is_computed() + + # Output of an Intrinsic action is the string representation of the output of the + # intrinsic. Parse the string. + result_str = model_output_thunk.value + if result_str is None: + raise ValueError("Model output is None.") + result_json = json.loads(result_str) + return result_json diff --git a/mellea/stdlib/components/intrinsic/core.py b/mellea/stdlib/components/intrinsic/core.py new file mode 100644 index 000000000..51d887ea2 --- /dev/null +++ b/mellea/stdlib/components/intrinsic/core.py @@ -0,0 +1,21 @@ +"""Intrinsic functions for core model capabilities.""" + +from ....backends.adapters import AdapterMixin +from ...context import ChatContext +from ._util import call_intrinsic as _call_intrinsic + + +def check_certainty(context: ChatContext, backend: AdapterMixin) -> float: + """Estimate the model's certainty about its last response. + + Intrinsic function that evaluates how certain the model is about the + assistant's response to a user's question. The context should end with + a user question followed by an assistant answer. + + :param context: Chat context containing user question and assistant answer. + :param backend: Backend instance that supports LoRA/aLoRA adapters. + + :return: Certainty score as a float (higher = more certain). + """ + result_json = _call_intrinsic("uncertainty", context, backend) + return result_json["certainty"] diff --git a/mellea/stdlib/components/intrinsic/rag.py b/mellea/stdlib/components/intrinsic/rag.py index 2d4962288..09810c063 100644 --- a/mellea/stdlib/components/intrinsic/rag.py +++ b/mellea/stdlib/components/intrinsic/rag.py @@ -1,15 +1,12 @@ """Intrinsic functions related to retrieval-augmented generation.""" import collections.abc -import json -from ....backends import ModelOption -from ....backends.adapters import AdapterMixin, AdapterType, GraniteCommonAdapter -from ....stdlib import functional as mfuncs +from ....backends.adapters import AdapterMixin from ...components import Document from ...context import ChatContext from ..chat import Message -from .intrinsic import Intrinsic +from ._util import call_intrinsic as _call_intrinsic _ANSWER_RELEVANCE_CORRECTION_METHODS = { "Excessive unnecessary information": "removing the excessive information from the " @@ -30,54 +27,6 @@ so it's important to stick to in-domain prompts.""" -def _call_intrinsic( - intrinsic_name: str, - context: ChatContext, - backend: AdapterMixin, - /, - kwargs: dict | None = None, -): - """Shared code for invoking intrinsics. - - :returns: Result of the call in JSON format. - """ - # Adapter needs to be present in the backend before it can be invoked. - # We must create the Adapter object in order to determine whether we need to create - # the Adapter object. - base_model_name = backend.base_model_name - if base_model_name is None: - raise ValueError("Backend has no model ID") - adapter = GraniteCommonAdapter( - intrinsic_name, adapter_type=AdapterType.LORA, base_model_name=base_model_name - ) - if adapter.qualified_name not in backend.list_adapters(): - backend.add_adapter(adapter) - - # Create the AST node for the action we wish to perform. - intrinsic = Intrinsic(intrinsic_name, intrinsic_kwargs=kwargs) - - # Execute the AST node. - model_output_thunk, _ = mfuncs.act( - intrinsic, - context, - backend, - model_options={ModelOption.TEMPERATURE: 0.0}, - # No rejection sampling, please - strategy=None, - ) - - # act() can return a future. Don't know how to handle one from non-async code. - assert model_output_thunk.is_computed() - - # Output of an Intrinsic action is the string representation of the output of the - # intrinsic. Parse the string. - result_str = model_output_thunk.value - if result_str is None: - raise ValueError("Model output is None.") - result_json = json.loads(result_str) - return result_json - - def check_answerability( question: str, documents: collections.abc.Iterable[Document], diff --git a/test/stdlib/components/intrinsic/test_core.py b/test/stdlib/components/intrinsic/test_core.py new file mode 100644 index 000000000..3b0ea82ab --- /dev/null +++ b/test/stdlib/components/intrinsic/test_core.py @@ -0,0 +1,77 @@ +"""Tests of the code in ``mellea.stdlib.intrinsics.core``""" + +import gc +import json +import os +import pathlib + +import pytest +import torch + +from mellea.backends.huggingface import LocalHFBackend +from mellea.stdlib.components import Message +from mellea.stdlib.components.intrinsic import core +from mellea.stdlib.context import ChatContext + +# Skip entire module in CI since all tests are qualitative +pytestmark = [ + pytest.mark.skipif( + int(os.environ.get("CICD", 0)) == 1, + reason="Skipping core intrinsic tests in CI - all qualitative tests", + ), + pytest.mark.huggingface, + pytest.mark.requires_gpu, + pytest.mark.requires_heavy_ram, + pytest.mark.llm, +] + +DATA_ROOT = pathlib.Path(os.path.dirname(__file__)) / "testdata" +"""Location of data files for the tests in this file.""" + + +BASE_MODEL = "ibm-granite/granite-4.0-micro" + + +@pytest.fixture(name="backend", scope="module") +def _backend(): + """Backend used by the tests in this file. Module-scoped to avoid reloading the 3B model for each test.""" + # Prevent thrashing if the default device is CPU + torch.set_num_threads(4) + + backend_ = LocalHFBackend(model_id=BASE_MODEL) + yield backend_ + + # Code after yield is cleanup code. + # Free GPU memory with extreme prejudice. + del backend_ + gc.collect() + gc.collect() + gc.collect() + torch.cuda.empty_cache() + + +def _read_input_json(file_name: str): + """Read test data from JSON and convert to a ChatContext.""" + with open(DATA_ROOT / "input_json" / file_name, encoding="utf-8") as f: + json_data = json.load(f) + + context = ChatContext() + for m in json_data["messages"]: + context = context.add(Message(m["role"], m["content"])) + return context + + +@pytest.mark.qualitative +def test_certainty(backend): + """Verify that the uncertainty/certainty intrinsic functions properly.""" + context = _read_input_json("uncertainty.json") + + result = core.check_certainty(context, backend) + assert 0.0 <= result <= 1.0 + + result2 = core.check_certainty(context, backend) + assert 0.0 <= result2 <= 1.0 + + +if __name__ == "__main__": + pytest.main([__file__]) diff --git a/test/stdlib/components/intrinsic/testdata/input_json/uncertainty.json b/test/stdlib/components/intrinsic/testdata/input_json/uncertainty.json new file mode 100644 index 000000000..045c0e667 --- /dev/null +++ b/test/stdlib/components/intrinsic/testdata/input_json/uncertainty.json @@ -0,0 +1,12 @@ +{ + "messages": [ + { + "role": "user", + "content": "What is the square root of 4?" + }, + { + "role": "assistant", + "content": "The square root of 4 is 2." + } + ] +} \ No newline at end of file From 610a836faaa05fcfa3f7d3f309bf061c1c9dffb5 Mon Sep 17 00:00:00 2001 From: inkpad Date: Tue, 17 Feb 2026 03:34:53 +0000 Subject: [PATCH 02/14] Fix uncertainty description in README Co-Authored-By: ink-pad --- docs/examples/intrinsics/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/examples/intrinsics/README.md b/docs/examples/intrinsics/README.md index 97c8dabf5..f993b6eb6 100644 --- a/docs/examples/intrinsics/README.md +++ b/docs/examples/intrinsics/README.md @@ -32,7 +32,7 @@ Detects when model outputs contain hallucinated information. Rewrites queries for better retrieval or understanding. ### uncertainty.py -Estimates the model's certainty about answering a question given retrieved documents. +Estimates the model's certainty about answering a question. ## Concepts Demonstrated From e2c438f9b4941065313abf0182220f8d2b90c031 Mon Sep 17 00:00:00 2001 From: inkpad Date: Tue, 17 Feb 2026 16:35:04 +0000 Subject: [PATCH 03/14] Rename _call_intrinsic to call_intrinsic MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Drop the underscore prefix and alias — use call_intrinsic consistently across _util.py, rag.py, core.py, and test_rag.py. Co-Authored-By: ink-pad --- mellea/stdlib/components/intrinsic/core.py | 4 ++-- mellea/stdlib/components/intrinsic/rag.py | 18 +++++++++--------- test/stdlib/components/intrinsic/test_rag.py | 2 +- 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/mellea/stdlib/components/intrinsic/core.py b/mellea/stdlib/components/intrinsic/core.py index 51d887ea2..1cad7e5b7 100644 --- a/mellea/stdlib/components/intrinsic/core.py +++ b/mellea/stdlib/components/intrinsic/core.py @@ -2,7 +2,7 @@ from ....backends.adapters import AdapterMixin from ...context import ChatContext -from ._util import call_intrinsic as _call_intrinsic +from ._util import call_intrinsic def check_certainty(context: ChatContext, backend: AdapterMixin) -> float: @@ -17,5 +17,5 @@ def check_certainty(context: ChatContext, backend: AdapterMixin) -> float: :return: Certainty score as a float (higher = more certain). """ - result_json = _call_intrinsic("uncertainty", context, backend) + result_json = call_intrinsic("uncertainty", context, backend) return result_json["certainty"] diff --git a/mellea/stdlib/components/intrinsic/rag.py b/mellea/stdlib/components/intrinsic/rag.py index 09810c063..f4ea07a62 100644 --- a/mellea/stdlib/components/intrinsic/rag.py +++ b/mellea/stdlib/components/intrinsic/rag.py @@ -6,7 +6,7 @@ from ...components import Document from ...context import ChatContext from ..chat import Message -from ._util import call_intrinsic as _call_intrinsic +from ._util import call_intrinsic _ANSWER_RELEVANCE_CORRECTION_METHODS = { "Excessive unnecessary information": "removing the excessive information from the " @@ -48,7 +48,7 @@ def check_answerability( :return: Answerability score as a floating-point value from 0 to 1. """ - result_json = _call_intrinsic( + result_json = call_intrinsic( "answerability", context.add(Message("user", question, documents=list(documents))), backend, @@ -71,7 +71,7 @@ def rewrite_question( :return: Rewritten version of ``question``. """ - result_json = _call_intrinsic( + result_json = call_intrinsic( "query_rewrite", context.add(Message("user", question)), backend ) return result_json["rewritten_question"] @@ -98,7 +98,7 @@ def clarify_query( :return: Clarification question string (e.g., "Do you mean A or B?"), or the string "CLEAR" if no clarification is needed """ - result_json = _call_intrinsic( + result_json = call_intrinsic( "query_clarification", context.add(Message("user", question, documents=list(documents))), backend, @@ -135,7 +135,7 @@ def find_citations( * ``citation_text`` Begin and end offsets are character offsets into their respective UTF-8 strings. """ - result_json = _call_intrinsic( + result_json = call_intrinsic( "citations", context.add(Message("assistant", response, documents=list(documents))), backend, @@ -160,7 +160,7 @@ def check_context_relevance( :return: Context relevance score as a floating-point value from 0 to 1. """ - result_json = _call_intrinsic( + result_json = call_intrinsic( "context_relevance", context.add(Message("user", question)), backend, @@ -196,7 +196,7 @@ def flag_hallucinated_content( * faithfulness_likelihood * explanation """ - result_json = _call_intrinsic( + result_json = call_intrinsic( "hallucination_detection", context.add(Message("assistant", response, documents=list(documents))), backend, @@ -232,7 +232,7 @@ def rewrite_answer_for_relevance( # * answer_relevance_analysis # * answer_relevance_category # * answer_relevance_likelihood - result_json = _call_intrinsic( + result_json = call_intrinsic( "answer_relevance_classifier", context.add(Message("assistant", response, documents=list(documents))), backend, @@ -248,7 +248,7 @@ def rewrite_answer_for_relevance( result_json["answer_relevance_category"] ] - result_json = _call_intrinsic( + result_json = call_intrinsic( "answer_relevance_rewriter", context.add(Message("assistant", response, documents=list(documents))), backend, diff --git a/test/stdlib/components/intrinsic/test_rag.py b/test/stdlib/components/intrinsic/test_rag.py index e56b6a47b..ec54a75dc 100644 --- a/test/stdlib/components/intrinsic/test_rag.py +++ b/test/stdlib/components/intrinsic/test_rag.py @@ -199,7 +199,7 @@ def test_answer_relevance_classifier(backend): """Verify that the first phase of the answer relevance flow behaves as expectee.""" context, answer, docs = _read_input_json("answer_relevance.json") - result_json = rag._call_intrinsic( + result_json = rag.call_intrinsic( "answer_relevance_classifier", context.add(Message("assistant", answer, documents=list(docs))), backend, From 6f55767c7d709d577c84a3984fbedd5da7665c0c Mon Sep 17 00:00:00 2001 From: manish-nagireddy Date: Thu, 12 Mar 2026 02:31:13 +0000 Subject: [PATCH 04/14] test --- test.txt | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 test.txt diff --git a/test.txt b/test.txt new file mode 100644 index 000000000..e69de29bb From bae9f29571572ca0145dd598bd3480f0ee009f54 Mon Sep 17 00:00:00 2001 From: manish-nagireddy Date: Thu, 12 Mar 2026 02:33:51 +0000 Subject: [PATCH 05/14] removed test file --- test.txt | 0 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 test.txt diff --git a/test.txt b/test.txt deleted file mode 100644 index e69de29bb..000000000 From 6566caf3e31af5828131e7b14e1c69a1d0993c72 Mon Sep 17 00:00:00 2001 From: manish-nagireddy Date: Thu, 12 Mar 2026 15:55:14 +0000 Subject: [PATCH 06/14] added req check intrinsic --- docs/examples/intrinsics/README.md | 5 +- docs/examples/intrinsics/requirement_check.py | 51 +++++++++++++++++++ mellea/backends/adapters/catalog.py | 4 +- mellea/stdlib/components/intrinsic/core.py | 29 +++++++++++ test/stdlib/components/intrinsic/test_core.py | 23 +++++++-- .../input_json/requirement_check.json | 13 +++++ 6 files changed, 119 insertions(+), 6 deletions(-) create mode 100644 docs/examples/intrinsics/requirement_check.py create mode 100644 test/stdlib/components/intrinsic/testdata/input_json/requirement_check.json diff --git a/docs/examples/intrinsics/README.md b/docs/examples/intrinsics/README.md index f993b6eb6..d70a55ff0 100644 --- a/docs/examples/intrinsics/README.md +++ b/docs/examples/intrinsics/README.md @@ -34,6 +34,9 @@ Rewrites queries for better retrieval or understanding. ### uncertainty.py Estimates the model's certainty about answering a question. +### requirement_check.py +Detect if text adheres to provided requirements. + ## Concepts Demonstrated - **Intrinsic Functions**: Specialized model capabilities beyond text generation @@ -51,7 +54,7 @@ from mellea.stdlib.components import Intrinsic import mellea.stdlib.functional as mfuncs # Create backend and adapter -backend = LocalHFBackend(model_id="ibm-granite/granite-3.3-8b-instruct") +backend = LocalHFBackend(model_id="ibm-granite/granite-4.0-micro") adapter = GraniteCommonAdapter("requirement_check", base_model_name=backend.base_model_name) backend.add_adapter(adapter) diff --git a/docs/examples/intrinsics/requirement_check.py b/docs/examples/intrinsics/requirement_check.py new file mode 100644 index 000000000..00df9944b --- /dev/null +++ b/docs/examples/intrinsics/requirement_check.py @@ -0,0 +1,51 @@ +# pytest: huggingface, requires_heavy_ram, llm + +"""Example usage of the requirement check intrinsic. + +Intrinsic function that determines if the text satisfies the given requirements. + +To run this script from the root of the Mellea source tree, use the command: +``` +uv run python docs/examples/intrinsics/requirement_check.py +``` +""" + +from mellea.backends.huggingface import LocalHFBackend +from mellea.stdlib.components import Message +from mellea.stdlib.components.intrinsic import core +from mellea.stdlib.context import ChatContext + +user_text = "Invite for an IBM office party." +response_text = """ +Dear Team, + +To celebrate our recent successes and take a well-deserved moment to recharge, +you are cordially invited to a team social. Please join us for an evening of +live music, appetizers, and drinks as we recognize our collective wins. + +Event Details +* **Date:** Saturday, April 25, 2026 +* **Time:** 6:00 PM +* **Location:** Ryan’s Bar, Chelsea, NY +* **Highlights:** Live entertainment and refreshments + +RSVP +To ensure we have an accurate headcount for catering, please confirm your +attendance by **Friday, April 10, 2026**. + +We look forward to seeing everyone there and celebrating our hard work together. + +**Best regards,** +[Your Name/Management Team] +""" +requirement = "Use a professional tone." + +backend = LocalHFBackend(model_id="ibm-granite/granite-4.0-micro") +context = ( + ChatContext() + .add(Message("user", user_text)) + .add(Message("assistant", response_text)) +) + +result = core.requirement_check(context, backend, requirement) +print(f"Requirements Satisfied: {result}") # float between 0.0 and 1.0 diff --git a/mellea/backends/adapters/catalog.py b/mellea/backends/adapters/catalog.py index 28ad0e929..fb10987cc 100644 --- a/mellea/backends/adapters/catalog.py +++ b/mellea/backends/adapters/catalog.py @@ -40,14 +40,14 @@ class IntriniscsCatalogEntry(pydantic.BaseModel): _RAG_REPO = "ibm-granite/granite-lib-rag-r1.0" _CORE_REPO = "ibm-granite/rag-intrinsics-lib" -_CORE_R1_REPO = "ibm-granite/granite-lib-core-r1.0" +_CORE_R1_REPO = "ibm-granite/granitelib-core-r1.0" _INTRINSICS_CATALOG_ENTRIES = [ ############################################ # Core Intrinsics ############################################ - IntriniscsCatalogEntry(name="requirement_check", repo_id=_CORE_REPO), + IntriniscsCatalogEntry(name="requirement_check", repo_id=_CORE_R1_REPO), IntriniscsCatalogEntry(name="uncertainty", repo_id=_CORE_R1_REPO), ############################################ # RAG Intrinsics diff --git a/mellea/stdlib/components/intrinsic/core.py b/mellea/stdlib/components/intrinsic/core.py index 1cad7e5b7..8f9afae10 100644 --- a/mellea/stdlib/components/intrinsic/core.py +++ b/mellea/stdlib/components/intrinsic/core.py @@ -1,6 +1,7 @@ """Intrinsic functions for core model capabilities.""" from ....backends.adapters import AdapterMixin +from ...components import Message from ...context import ChatContext from ._util import call_intrinsic @@ -19,3 +20,31 @@ def check_certainty(context: ChatContext, backend: AdapterMixin) -> float: """ result_json = call_intrinsic("uncertainty", context, backend) return result_json["certainty"] + + +_EVALUATION_PROMPT = ( + "Please verify if the assistant's generation satisfies the user's " + "requirements or not and reply with a binary label accordingly. " + 'Respond with a json {"score": "yes"} if the constraints are ' + 'satisfied or respond with {"score": "no"} if the constraints are not ' + "satisfied." +) + + +def requirement_check(context: ChatContext, backend: AdapterMixin, requirement: str) -> float: + """Detect if text adheres to provided requirements. + + Intrinsic function that determines if the text satisfies the given + requirements. Appends an evaluation prompt to the context following + the format specified by the Granite Guardian requirement checker model card. + + :param context: Chat context containing user question and assistant answer. + :param backend: Backend instance that supports LoRA/aLoRA adapters. + :param requirement: set of requirements to satisfy + + :return: Score as a float between 0.0 and 1.0 (higher = more likely satisfied). + """ + eval_message = f": {requirement}\n{_EVALUATION_PROMPT}" + context = context.add(Message("user", eval_message)) + result_json = call_intrinsic("requirement_check", context, backend) + return result_json["requirement_check"]["score"] diff --git a/test/stdlib/components/intrinsic/test_core.py b/test/stdlib/components/intrinsic/test_core.py index 3b0ea82ab..b3fa8feb6 100644 --- a/test/stdlib/components/intrinsic/test_core.py +++ b/test/stdlib/components/intrinsic/test_core.py @@ -51,20 +51,24 @@ def _backend(): def _read_input_json(file_name: str): - """Read test data from JSON and convert to a ChatContext.""" + """Read test data from JSON and convert to a ChatContext. + + Returns the context and the raw JSON data (for accessing extra fields + like ``requirement``). + """ with open(DATA_ROOT / "input_json" / file_name, encoding="utf-8") as f: json_data = json.load(f) context = ChatContext() for m in json_data["messages"]: context = context.add(Message(m["role"], m["content"])) - return context + return context, json_data @pytest.mark.qualitative def test_certainty(backend): """Verify that the uncertainty/certainty intrinsic functions properly.""" - context = _read_input_json("uncertainty.json") + context, _ = _read_input_json("uncertainty.json") result = core.check_certainty(context, backend) assert 0.0 <= result <= 1.0 @@ -73,5 +77,18 @@ def test_certainty(backend): assert 0.0 <= result2 <= 1.0 +@pytest.mark.qualitative +def test_requirement_check(backend): + """Verify that the requirement check intrinsic functions properly.""" + context, json_data = _read_input_json("requirement_check.json") + requirement = json_data["requirement"] + + result = core.requirement_check(context, backend, requirement) + assert 0.0 <= result <= 1.0 + + result2 = core.requirement_check(context, backend, requirement) + assert 0.0 <= result2 <= 1.0 + + if __name__ == "__main__": pytest.main([__file__]) diff --git a/test/stdlib/components/intrinsic/testdata/input_json/requirement_check.json b/test/stdlib/components/intrinsic/testdata/input_json/requirement_check.json new file mode 100644 index 000000000..e8c2065d8 --- /dev/null +++ b/test/stdlib/components/intrinsic/testdata/input_json/requirement_check.json @@ -0,0 +1,13 @@ +{ + "messages": [ + { + "role": "user", + "content": "Invite for an IBM office party." + }, + { + "role": "assistant", + "content": "\nDear Team,\n\nTo celebrate our recent successes and take a well-deserved moment to recharge,\nyou are cordially invited to a team social. Please join us for an evening of\nlive music, appetizers, and drinks as we recognize our collective wins.\n\nEvent Details\n* **Date:** Saturday, April 25, 2026\n* **Time:** 6:00 PM\n* **Location:** Ryan's Bar, Chelsea, NY\n* **Highlights:** Live entertainment and refreshments\n\nRSVP\nTo ensure we have an accurate headcount for catering, please confirm your\nattendance by **Friday, April 10, 2026**.\n\nWe look forward to seeing everyone there and celebrating our hard work together.\n\n**Best regards,**\n[Your Name/Management Team]\n" + } + ], + "requirement": "Use a professional tone." +} From b6c9b883eae4575e7ab13e392c32938e81ce3c50 Mon Sep 17 00:00:00 2001 From: Dennis Wei Date: Fri, 13 Mar 2026 17:32:56 -0700 Subject: [PATCH 07/14] feat: extend sentence boundary marking to conversation history with shared index - Add `index` parameter to `mark_sentence_boundaries()` to allow callers to continue numbering across multiple calls; return the next available index - Add `all_but_last_message` as a valid `sentence_boundaries` key - Extend `_mark_sentence_boundaries()` to tag prior conversation turns when `all_but_last_message` is configured, using a shared running index with documents so that each context sentence has a globally unique tag Co-Authored-By: Claude Sonnet 4.6 Signed-off-by: Dennis Wei --- mellea/formatters/granite/intrinsics/input.py | 57 +++++++++++++++---- 1 file changed, 46 insertions(+), 11 deletions(-) diff --git a/mellea/formatters/granite/intrinsics/input.py b/mellea/formatters/granite/intrinsics/input.py index 7ce05708c..a9b515a41 100644 --- a/mellea/formatters/granite/intrinsics/input.py +++ b/mellea/formatters/granite/intrinsics/input.py @@ -46,8 +46,8 @@ def sentence_delimiter(tag, sentence_num) -> str: def mark_sentence_boundaries( - split_strings: list[list[str]], tag_prefix: str -) -> list[str]: + split_strings: list[list[str]], tag_prefix: str, index: int = 0 +) -> tuple[list[str], int]: """Modify input strings by inserting sentence boundary markers. Modify one or more input strings by inserting a tag in the form @@ -57,10 +57,12 @@ def mark_sentence_boundaries( :param split_strings: Input string(s), pre-split into sentences :param tag_prefix: String to place before the number part of each tagged sentence boundary. + :param index: Starting index for sentence numbering. Defaults to 0. Pass a + non-zero value to continue numbering from a prior call. - :returns: List of input strings with all sentence boundaries marked. + :returns: Tuple of (list of input strings with all sentence boundaries marked, + next available index after the last sentence). """ - index = 0 result: list[str] = [] for sentences in split_strings: to_concat = [] @@ -68,7 +70,7 @@ def mark_sentence_boundaries( to_concat.append(f"{sentence_delimiter(tag_prefix, index)}{sentence}") index += 1 result.append(" ".join(to_concat)) - return result + return result, index def move_documents_to_message( @@ -258,10 +260,11 @@ def __init__( f"Received {self.sentence_boundaries}." ) for k, v in self.sentence_boundaries.items(): - if k not in ("last_message", "documents"): + if k not in ("last_message", "documents", "all_but_last_message"): raise ValueError( f"Unexpected location '{k}' in 'sentence_boundaries' field. " - f"Value should be 'last_message' or 'documents'." + f"Value should be 'last_message', 'documents', or " + f"'all_but_last_message'." ) if not isinstance(v, str): raise TypeError( @@ -291,6 +294,8 @@ def _mark_sentence_boundaries( :rtype: ChatCompletion """ # Mark sentence boundaries in the last message. + # last_message uses its own numbering starting from 0, independent of + # the numbering used for documents and conversation history. if self.sentence_boundaries and "last_message" in self.sentence_boundaries: messages = chat_completion.messages.copy() # Do not modify input! last_message_as_sentences = list( @@ -298,14 +303,17 @@ def _mark_sentence_boundaries( ) last_message_tag = self.sentence_boundaries["last_message"] if last_message_tag: - rewritten_last_message_text = mark_sentence_boundaries( + rewritten_texts, _ = mark_sentence_boundaries( [last_message_as_sentences], last_message_tag - )[0] - messages[-1].content = rewritten_last_message_text + ) + messages[-1].content = rewritten_texts[0] chat_completion = chat_completion.model_copy( update={"messages": messages} ) + # documents and all_but_last_message share a continuous numbering. + index = 0 + # Mark sentence boundaries in documents if present if ( chat_completion.extra_body @@ -322,11 +330,14 @@ def _mark_sentence_boundaries( # where `k` is the number of sentences in ALL documents. documents_tag = self.sentence_boundaries["documents"] if documents_tag: + rewritten_texts, index = mark_sentence_boundaries( + docs_as_sentences, documents_tag, index + ) rewritten_docs = [ doc.model_copy(update={"text": text}) for doc, text in zip( chat_completion.extra_body.documents, - mark_sentence_boundaries(docs_as_sentences, documents_tag), + rewritten_texts, strict=True, ) ] @@ -337,6 +348,30 @@ def _mark_sentence_boundaries( chat_completion = chat_completion.model_copy( update={"extra_body": extra_body} ) + + # Mark sentence boundaries in conversation history if requested. + # Uses the same numbering as documents, continuing from where they left off. + if ( + self.sentence_boundaries + and "all_but_last_message" in self.sentence_boundaries + ): + history_tag = self.sentence_boundaries["all_but_last_message"] + if history_tag: + messages = chat_completion.messages.copy() # Do not modify input! + for i, message in enumerate(messages[:-1]): + msg_as_sentences = list( + self.sentence_splitter.tokenize(message.content) + ) + rewritten_texts, index = mark_sentence_boundaries( + [msg_as_sentences], history_tag, index + ) + messages[i] = message.model_copy( + update={"content": rewritten_texts[0]} + ) + chat_completion = chat_completion.model_copy( + update={"messages": messages} + ) + return chat_completion def _transform( From b12c7a2a96a65213e2a7a9def614db45b7199879 Mon Sep 17 00:00:00 2001 From: Dennis Wei Date: Fri, 13 Mar 2026 18:10:06 -0700 Subject: [PATCH 08/14] feat: extend DecodeSentences to conversation history and multi-source decoding - Accept `source: str | list[str]` to allow a single DecodeSentences rule to decode sentences from multiple locations in one pass - Add `all_but_last_message` as a valid source, decoding prior conversation turns with a running sentence index shared across all sources - Add optional `message_index` output field that records which conversation turn each attributed sentence came from Co-Authored-By: Claude Sonnet 4.6 Signed-off-by: Dennis Wei --- .../formatters/granite/intrinsics/output.py | 181 +++++++++++------- 1 file changed, 114 insertions(+), 67 deletions(-) diff --git a/mellea/formatters/granite/intrinsics/output.py b/mellea/formatters/granite/intrinsics/output.py index 5eb0bafb3..89db9859c 100644 --- a/mellea/formatters/granite/intrinsics/output.py +++ b/mellea/formatters/granite/intrinsics/output.py @@ -461,23 +461,27 @@ def __init__( config: dict, input_path_expr: list[str | int | None], /, - source: str, + source: str | list[str], output_names: dict, ): """Initialize DecodeSentences transformation rule. - :param source: Name of the location to look for sentences; can be "last_message" - or "documents". + :param source: Name (or list of names) of the location(s) to look for + sentences; each name can be "last_message", "documents", or + "all_but_last_message". :param output_names: Names of new result fields to add """ super().__init__(config, input_path_expr) - allowed_sources = ("last_message", "documents") - if source not in allowed_sources: - raise ValueError( - f"'source' argument must be one of {allowed_sources}. " - f"Received '{source}'" - ) + if isinstance(source, str): + source = [source] + allowed_sources = ("last_message", "documents", "all_but_last_message") + for s in source: + if s not in allowed_sources: + raise ValueError( + f"'source' argument must be one of {allowed_sources}. " + f"Received '{s}'" + ) self.source = source if not isinstance(output_names, dict): @@ -485,7 +489,9 @@ def __init__( f"Expected mapping for output_names, but received {output_names}" ) for k in output_names: - if source == "documents" and k == "document_id": + if "documents" in source and k == "document_id": + continue + if "all_but_last_message" in source and k == "message_index": continue if k not in ("begin", "end", "text"): raise ValueError(f"Unexpected key '{k}' in output_names") @@ -495,6 +501,7 @@ def __init__( self.end_name = output_names.get("end") self.text_name = output_names.get("text") self.document_id_name = output_names.get("document_id") + self.message_index_name = output_names.get("message_index") if config["docs_as_message"] and config["docs_as_message"] not in [ "json", @@ -519,77 +526,114 @@ def _prepare( f"'{self.rule_name()}' rule requires this object." ) - if self.source == "documents": - tag = self.config["sentence_boundaries"]["documents"] - if tag is None: - raise ValueError( - f"'{self.rule_name()}' attempting to decode document sentences, " - f"but 'sentence_boundaries' section of config file is missing " - f"the entry that tells how to tag document sentence boundaries." - ) + begins: list[int] = [] + ends: list[int] = [] + texts: list[str] = [] + document_ids: list[str | None] = [] + message_indices: list[int | None] = [] + next_sentence_num = 0 + + for src in self.source: + if src == "documents": + tag = self.config["sentence_boundaries"]["documents"] + if tag is None: + raise ValueError( + f"'{self.rule_name()}' attempting to decode document sentences, " + f"but 'sentence_boundaries' section of config file is missing " + f"the entry that tells how to tag document sentence boundaries." + ) - documents: list[Document] = [] - if not self.config["docs_as_message"]: - # Most common path: Documents from extra_body - if chat_completion.extra_body is not None: - documents = chat_completion.extra_body.documents or [] - else: - # Model requires documents in a user message. Decode the message. - if self.config["docs_as_message"] == "json": - documents_json = json.loads(chat_completion.messages[0].content) - documents = [Document.model_validate(d) for d in documents_json] - elif self.config["docs_as_message"] == "roles": - for message in chat_completion.messages: - if message.role.startswith("document "): - document = Document( - doc_id=message.role[len("document ") :], - text=message.content, - ) - documents.append(document) + documents: list[Document] = [] + if not self.config["docs_as_message"]: + # Most common path: Documents from extra_body + if chat_completion.extra_body is not None: + documents = chat_completion.extra_body.documents or [] else: + # Model requires documents in a user message. Decode the message. + if self.config["docs_as_message"] == "json": + documents_json = json.loads(chat_completion.messages[0].content) + documents = [Document.model_validate(d) for d in documents_json] + elif self.config["docs_as_message"] == "roles": + for message in chat_completion.messages: + if message.role.startswith("document "): + document = Document( + doc_id=message.role[len("document ") :], + text=message.content, + ) + documents.append(document) + else: + raise ValueError( + f"Unsupported doc type {self.config['docs_as_message']}" + ) + + # De-split sentences; numbers start at next_sentence_num and continue + # across documents. + for d in documents: + local_results = _desplit_sentences(d.text, tag, next_sentence_num) + num_local_sentences = len(local_results["begins"]) + begins.extend(local_results["begins"]) + ends.extend(local_results["ends"]) + texts.extend(local_results["texts"]) + document_ids.extend([d.doc_id] * num_local_sentences) + message_indices.extend([None] * num_local_sentences) + next_sentence_num += num_local_sentences + + elif src == "last_message": + tag = self.config["sentence_boundaries"]["last_message"] + if tag is None: raise ValueError( - f"Unsupported doc type {self.config['docs_as_message']}" + f"'{self.rule_name()}' attempting to decode the last message, " + f"but 'sentence_boundaries' section of config file is missing " + f"the entry that tells how to tag message sentence boundaries." ) - # De-split the sentences in each document in turn. Sentence numbers - # start at zero on the first document and continue in subsequent documents. - begins = [] - ends = [] - texts = [] - document_ids = [] - - next_sentence_num = 0 - for d in documents: - local_results = _desplit_sentences(d.text, tag, next_sentence_num) + # Use second-to-last turn if the input processing added an instruction turn + message_ix = -2 if self.config["instruction"] else -1 + target_text = chat_completion.messages[message_ix].content + local_results = _desplit_sentences(target_text, tag, next_sentence_num) num_local_sentences = len(local_results["begins"]) begins.extend(local_results["begins"]) ends.extend(local_results["ends"]) texts.extend(local_results["texts"]) - document_ids.extend([d.doc_id] * num_local_sentences) + document_ids.extend([None] * num_local_sentences) + message_indices.extend([None] * num_local_sentences) next_sentence_num += num_local_sentences - return { - "begins": begins, - "ends": ends, - "texts": texts, - "document_ids": document_ids, - } - if self.source == "last_message": - tag = self.config["sentence_boundaries"]["last_message"] - if tag is None: - raise ValueError( - f"'{self.rule_name()}' attempting to decode the last message, " - f"but 'sentence_boundaries' section of config file is missing " - f"the entry that tells how to tag message sentence boundaries." - ) + elif src == "all_but_last_message": + tag = self.config["sentence_boundaries"]["all_but_last_message"] + if tag is None: + raise ValueError( + f"'{self.rule_name()}' attempting to decode conversation " + f"history sentences, but 'sentence_boundaries' section of " + f"config file is missing the entry that tells how to tag " + f"all_but_last_message sentence boundaries." + ) - # Use second-to-last turn if the input processing added an instruction turn - message_ix = -2 if self.config["instruction"] else -1 - target_text = chat_completion.messages[message_ix].content + # Use second-to-last as the boundary if an instruction turn was added + last_ix = -2 if self.config["instruction"] else -1 + history_messages = chat_completion.messages[:last_ix] + for i, message in enumerate(history_messages): + local_results = _desplit_sentences( + message.content, tag, next_sentence_num + ) + num_local_sentences = len(local_results["begins"]) + begins.extend(local_results["begins"]) + ends.extend(local_results["ends"]) + texts.extend(local_results["texts"]) + document_ids.extend([None] * num_local_sentences) + message_indices.extend([i] * num_local_sentences) + next_sentence_num += num_local_sentences - return _desplit_sentences(target_text, tag, 0) + else: + raise ValueError(f"Unexpected source string '{src}'") - raise ValueError(f"Unexpected source string '{self.source}'") + return { + "begins": begins, + "ends": ends, + "texts": texts, + "document_ids": document_ids, + "message_indices": message_indices, + } def _transform(self, value: Any, path: tuple, prepare_output: dict) -> dict: # Unpack global values we set aside during the prepare phase @@ -597,6 +641,7 @@ def _transform(self, value: Any, path: tuple, prepare_output: dict) -> dict: ends = prepare_output["ends"] texts = prepare_output["texts"] document_ids = prepare_output.get("document_ids") + message_indices = prepare_output.get("message_indices") if not isinstance(value, int): raise TypeError( @@ -614,6 +659,8 @@ def _transform(self, value: Any, path: tuple, prepare_output: dict) -> dict: result[self.text_name] = texts[sentence_num] if self.document_id_name is not None: result[self.document_id_name] = document_ids[sentence_num] # type: ignore[index] + if self.message_index_name is not None: + result[self.message_index_name] = message_indices[sentence_num] # type: ignore[index] return result From 3a36ffcd91eb26906ad2815ac8ac3b27e663e156 Mon Sep 17 00:00:00 2001 From: Dennis Wei Date: Fri, 13 Mar 2026 18:29:01 -0700 Subject: [PATCH 09/14] feat: add context-attribution catalog entry and update core repo - Update _CORE_REPO to "ibm-granite/granitelib-core-r1.0" - Add context-attribution intrinsic pointing to _CORE_REPO Co-Authored-By: Claude Sonnet 4.6 Signed-off-by: Dennis Wei --- mellea/backends/adapters/catalog.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/mellea/backends/adapters/catalog.py b/mellea/backends/adapters/catalog.py index 99d773ca4..2f8857732 100644 --- a/mellea/backends/adapters/catalog.py +++ b/mellea/backends/adapters/catalog.py @@ -39,13 +39,14 @@ class IntriniscsCatalogEntry(pydantic.BaseModel): _RAG_REPO = "ibm-granite/granite-lib-rag-r1.0" -_CORE_REPO = "ibm-granite/rag-intrinsics-lib" +_CORE_REPO = "ibm-granite/granitelib-core-r1.0" _INTRINSICS_CATALOG_ENTRIES = [ ############################################ # Core Intrinsics ############################################ + IntriniscsCatalogEntry(name="context-attribution", repo_id=_CORE_REPO), IntriniscsCatalogEntry(name="requirement_check", repo_id=_CORE_REPO), IntriniscsCatalogEntry(name="uncertainty", repo_id=_CORE_REPO), ############################################ From 4b3630c18fb26c38970ae9aa2aaa993f830234c1 Mon Sep 17 00:00:00 2001 From: Dennis Wei Date: Mon, 16 Mar 2026 00:25:40 -0700 Subject: [PATCH 10/14] test: add context-attribution test data and formatter tests - Add input/test_canned_input/test_canned_output/expected_result JSON test data files - Add YamlJsonCombo entry for context-attribution pointing to ibm-granite/granitelib-core-r1.0 - Exclude context-attribution from Ollama inference tests via _NO_OLLAMA_ADAPTER since an Ollama LoRA adapter is not yet available on the HF Hub Co-Authored-By: Claude Sonnet 4.6 Signed-off-by: Dennis Wei --- .../granite/test_intrinsics_formatters.py | 11 +++ .../input_json/context-attribution.json | 37 +++++++++ .../context-attribution.json | 76 +++++++++++++++++++ .../expected_result/context-attribution.json | 30 ++++++++ .../model_output/context-attribution.json | 35 +++++++++ .../test_run_ollama/context-attribution.json | 11 +++ .../context-attribution.json | 11 +++ 7 files changed, 211 insertions(+) create mode 100644 test/formatters/granite/testdata/input_json/context-attribution.json create mode 100644 test/formatters/granite/testdata/test_canned_input/context-attribution.json create mode 100644 test/formatters/granite/testdata/test_canned_output/expected_result/context-attribution.json create mode 100644 test/formatters/granite/testdata/test_canned_output/model_output/context-attribution.json create mode 100644 test/formatters/granite/testdata/test_run_ollama/context-attribution.json create mode 100644 test/formatters/granite/testdata/test_run_transformers/context-attribution.json diff --git a/test/formatters/granite/test_intrinsics_formatters.py b/test/formatters/granite/test_intrinsics_formatters.py index 07b41d447..1359494e8 100644 --- a/test/formatters/granite/test_intrinsics_formatters.py +++ b/test/formatters/granite/test_intrinsics_formatters.py @@ -231,6 +231,12 @@ def _maybe_download_yaml(self): # task="citations", # is_alora=True, # ), + YamlJsonCombo( + short_name="context-attribution", + inputs_file=_INPUT_JSON_DIR / "context-attribution.json", + task="context-attribution", + repo_id="ibm-granite/granitelib-core-r1.0", + ), # gpt-oss-20b intrinsics (canned output tests only, no inference) YamlJsonCombo( short_name="gpt_oss_answerability", @@ -287,12 +293,17 @@ def _maybe_download_yaml(self): } # Combinations suitable for an Ollama backend +_NO_OLLAMA_ADAPTER = { + # Ollama LoRA adapter not yet available on HF Hub + "context-attribution" +} _YAML_JSON_COMBOS_FOR_OLLAMA = { k: v for k, v in _YAML_JSON_COMBOS.items() if v.task is not None and not v.is_alora and v.base_model_id == "ibm-granite/granite-4.0-micro" + and k not in _NO_OLLAMA_ADAPTER } diff --git a/test/formatters/granite/testdata/input_json/context-attribution.json b/test/formatters/granite/testdata/input_json/context-attribution.json new file mode 100644 index 000000000..1da0ad738 --- /dev/null +++ b/test/formatters/granite/testdata/input_json/context-attribution.json @@ -0,0 +1,37 @@ +{ + "messages": [ + { + "role": "user", + "content": "Who were the members of The Metal Ono Band, which was formed by Yoko Ono in 1976 to explore her interest in heavy metal music?" + }, + { + "role": "assistant", + "content": "I'm sorry, but I don't have the data to answer that specific question. " + }, + { + "role": "user", + "content": "What was the concept behind the formation of the Plastic Ono Band?" + }, + { + "role": "assistant", + "content": "The Plastic Ono Band was formed by John Lennon and Yoko Ono in 1969 as a collaborative vehicle for their artistic and personal projects. They decided to credit all their future efforts to this conceptual and collaborative group after their marriage in 1969." + } + ], + "temperature": 0.0, + "extra_body": { + "documents": [ + { + "doc_id": "0", + "text": "The Plastic Ono Band is a band formed by John Lennon and Yoko Ono in 1969 as a vehicle for their collaborative and solo projects. Lennon and Ono had begun a personal and artistic relationship in 1968, collaborating on several experimental releases. Following their marriage in 1969, they decided that all of their future endeavours would be credited to a conceptual and collaborative vehicle, Plastic Ono Band. The band would go on to feature a rotating lineup of many musicians, including Eric Clapton, Klaus Voormann, Alan White, Billy Preston, Jim Keltner, Delaney & Bonnie and Friends, and Lennon's former Beatles bandmates George Harrison and Ringo Starr. Lennon and Ono left the UK to settle in New York City during the fall of 1971. In Greenwich Village, the couple became more politically active and began writing protest songs. These songs became the basis for their next album, Some Time in New York City. As backing, they enlisted the help of New York band Elephant's Memory, consisting of guitarist Wayne 'Tex' Gabriel, bassist Gary Van Scyoc, saxophonist Stan Bronstein, keyboardist Adam Ippolito, keyboardist John La Boosca, and drummer Richard Frank, Jr. Phil Spector produced, and Jim Keltner also played on the album. The album was released on 12 June 1972, credited to \"John & Yoko/Plastic Ono Band with Elephant's Memory plus Invisible Strings\". Some Time in New York City included a second disc, entitled Live Jam, which included the recordings from the 1969 Peace for Christmas concert and the 1971 performance with Frank Zappa. Ono and Lennon continued their work with Elephant's Memory throughout 1972, performing as the Plastic Ono Elephant's Memory Band (which also included Jim Keltner). On 30 August, they performed a pair of benefit concerts at Madison Square Garden. The benefit, entitled \"One to One\", was organised by Geraldo Rivera to raise money for children with mental challenges. By this time, La Boosca had departed the band, and the concert saw the addition of John Ward on bass. The concert was filmed and recorded, later released in February 1986 as the album Live In New York City. They also performed at the Jerry Lewis MDA Labor Day Telethon. The last collaboration of the Plastic Ono Elephant's Memory Band was Ono's double album Approximately Infinite Universe. It was recorded throughout the fall of 1972, and was released in January 1973." + }, + { + "doc_id": "1", + "text": "The Plastic Ono Band is a band formed by John Lennon and Yoko Ono in 1969 as a vehicle for their collaborative and solo projects. Lennon and Ono had begun a personal and artistic relationship in 1968, collaborating on several experimental releases. Following their marriage in 1969, they decided that all of their future endeavours would be credited to a conceptual and collaborative vehicle, Plastic Ono Band. The band would go on to feature a rotating lineup of many musicians, including Eric Clapton, Klaus Voormann, Alan White, Billy Preston, Jim Keltner, Delaney & Bonnie and Friends, and Lennon's former Beatles bandmates George Harrison and Ringo Starr. By the beginning of 1973, recording had begun in earnest on Ono's next album, Feeling the Space, featuring a new group of studio musicians. The newest incarnation of the Plastic Ono Band featured guitarist David Spinozza, keyboardist Ken Ascher, bassist Gordon Edwards, percussionists Arthur Jenkins and David Friedman, saxophonist Michael Brecker, pedal steel guitarist Sneaky Pete Kleinow, as well as regular contributor Jim Keltner. The album would be released in November. Throughout 1973, Lennon and Ono's relationship became strained. By August, the two had begun a period of separation that Lennon called \"The Lost Weekend\". Lennon began the recording of his own album, Mind Games, using the same players as on Feeling the Space, dubbed \"The Plastic U.F.Ono Band\". Around the time of the album's release in November, Lennon moved to Los Angeles with new lover May Pang. In October, Lennon began the recording of an album of rock 'n' roll oldies (a contractual obligation due to a lawsuit). These featured many Plastic Ono Band regulars (including much of the \"U.F.Ono Band\", Klaus Voorman, and the return of Phil Spector to the production chair), but upon release in 1975 as Rock 'n' Roll, it was credited to Lennon alone. The sessions for Rock 'n' Roll were extremely troubled, and the sessions were abandoned until a later date. In July 1974, Lennon returned to New York to record Walls and Bridges. The new \"Plastic Ono Nuclear Band\" featured both old and new faces, with Jim Keltner, Kenneth Ascher, and Arthur Jenkins continuing from Mind Games, the returns of Klaus Voorman, Nicky Hopkins, and Bobby Keys, and the addition of guitarists Jesse Ed Davis and Eddie Mottau. Recording was finished in August, and the album was released 26 September and 4 October in the US and UK respectively. Walls and Bridges would prove to be the last release of new material by the Plastic Ono Band in the 1970s. Lennon subsequently returned to his marriage with Ono and retired from music following the birth of his son Sean. The compilation Shaved Fish was released in October 1975, Lennon's last release credited to the Plastic Ono Band. Upon his and Ono's return to music in 1980 for the album Double Fantasy, they played with an all-new group of studio musicians who were not billed as any variation of the Plastic Ono Band name. Lennon was shot and killed shortly after the release of the album." + }, + { + "doc_id": "2", + "text": "John Winston Ono Lennon (9 October 1940 - 8 December 1980) was an English singer, songwriter, and peace activist who co-founded the Beatles, the most commercially successful band in the history of popular music. He and fellow member Paul McCartney formed a much-celebrated songwriting partnership. Along with George Harrison and Ringo Starr, the group would ascend to world-wide fame during the 1960s. During his marriage to Cynthia, Lennon's first son Julian was born at the same time that his commitments with the Beatles were intensifying at the height of Beatlemania. Lennon was touring with the Beatles when Julian was born on 8 April 1963. Julian's birth, like his mother Cynthia's marriage to Lennon, was kept secret because Epstein was convinced that public knowledge of such things would threaten the Beatles' commercial success. Julian recalled that as a small child in Weybridge some four years later, \"I was trundled home from school and came walking up with one of my watercolour paintings. It was just a bunch of stars and this blonde girl I knew at school. And Dad said, 'What's this?' I said, 'It's Lucy in the sky with diamonds.'\" Lennon used it as the title of a Beatles song, and though it was later reported to have been derived from the initials LSD, Lennon insisted, \"It's not an acid song.\" McCartney corroborated Lennon's explanation that Julian innocently came up with the name. Lennon was distant from Julian, who felt closer to McCartney than to his father. During a car journey to visit Cynthia and Julian during Lennon's divorce, McCartney composed a song, \"Hey Jules\", to comfort him. It would evolve into the Beatles song \"Hey Jude\". Lennon later said, \"That's his best song. It started off as a song about my son Julian ... he turned it into 'Hey Jude'. I always thought it was about me and Yoko but he said it wasn't.\" Lennon's relationship with Julian was already strained, and after Lennon and Ono moved to Manhattan in 1971, Julian would not see his father again until 1973. With Pang's encouragement, arrangements were made for Julian (and his mother) to visit Lennon in Los Angeles, where they went to Disneyland. Julian started to see his father regularly, and Lennon gave him a drumming part on a Walls and Bridges track. He bought Julian a Gibson Les Paul guitar and other instruments, and encouraged his interest in music by demonstrating guitar chord techniques. Julian recalls that he and his father \"got on a great deal better\" during the time he spent in New York: \"We had a lot of fun, laughed a lot and had a great time in general.\" In a Playboy interview with David Sheff shortly before his death, Lennon said, \"Sean was a planned child, and therein lies the difference. I don't love Julian any less as a child. He's still my son, whether he came from a bottle of whiskey or because they didn't have pills in those days. He's here, he belongs to me, and he always will.\" He said he was trying to re-establish a connection with the then 17-year-old, and confidently predicted, \"Julian and I will have a relationship in the future.\" After his death it was revealed that he had left Julian very little in his will." + } + ] + } +} \ No newline at end of file diff --git a/test/formatters/granite/testdata/test_canned_input/context-attribution.json b/test/formatters/granite/testdata/test_canned_input/context-attribution.json new file mode 100644 index 000000000..977ca9d4f --- /dev/null +++ b/test/formatters/granite/testdata/test_canned_input/context-attribution.json @@ -0,0 +1,76 @@ +{ + "messages": [ + { + "content": " Who were the members of The Metal Ono Band, which was formed by Yoko Ono in 1976 to explore her interest in heavy metal music?", + "role": "user" + }, + { + "content": " I'm sorry, but I don't have the data to answer that specific question.", + "role": "assistant" + }, + { + "content": " What was the concept behind the formation of the Plastic Ono Band?", + "role": "user" + }, + { + "content": " The Plastic Ono Band was formed by John Lennon and Yoko Ono in 1969 as a collaborative vehicle for their artistic and personal projects. They decided to credit all their future efforts to this conceptual and collaborative group after their marriage in 1969.", + "role": "assistant" + }, + { + "content": "You provided the last assistant response above based on context, which may include documents and/or previous conversation turns. Your response is divided into sentences, numbered in the format sentence 0 sentence 1 ... Sentences in the context are also numbered: sentence 0 sentence 1 ... For each response sentence, please list the context sentences that were most important for you to generate the response sentence. Provide your answer in JSON format, as an array of JSON objects, where each object has two members: \"r\" with the response sentence number as the value, and \"c\" with an array of context sentence numbers as the value. An example of such an array of objects is [{\"r\": 0, \"c\": [3, 1, 4]}, {\"r\": 1, \"c\": [1, 5]}]. List the context sentences in order from most important to least important. Ensure that you include an object for each response sentence, even if the corresponding array of context sentence numbers is empty. Answer with only the JSON and do not explain.\n", + "role": "user" + } + ], + "extra_body": { + "documents": [ + { + "text": " The Plastic Ono Band is a band formed by John Lennon and Yoko Ono in 1969 as a vehicle for their collaborative and solo projects. Lennon and Ono had begun a personal and artistic relationship in 1968, collaborating on several experimental releases. Following their marriage in 1969, they decided that all of their future endeavours would be credited to a conceptual and collaborative vehicle, Plastic Ono Band. The band would go on to feature a rotating lineup of many musicians, including Eric Clapton, Klaus Voormann, Alan White, Billy Preston, Jim Keltner, Delaney & Bonnie and Friends, and Lennon's former Beatles bandmates George Harrison and Ringo Starr. Lennon and Ono left the UK to settle in New York City during the fall of 1971. In Greenwich Village, the couple became more politically active and began writing protest songs. These songs became the basis for their next album, Some Time in New York City. As backing, they enlisted the help of New York band Elephant's Memory, consisting of guitarist Wayne 'Tex' Gabriel, bassist Gary Van Scyoc, saxophonist Stan Bronstein, keyboardist Adam Ippolito, keyboardist John La Boosca, and drummer Richard Frank, Jr. Phil Spector produced, and Jim Keltner also played on the album. The album was released on 12 June 1972, credited to \"John & Yoko/Plastic Ono Band with Elephant's Memory plus Invisible Strings\". Some Time in New York City included a second disc, entitled Live Jam, which included the recordings from the 1969 Peace for Christmas concert and the 1971 performance with Frank Zappa. Ono and Lennon continued their work with Elephant's Memory throughout 1972, performing as the Plastic Ono Elephant's Memory Band (which also included Jim Keltner). On 30 August, they performed a pair of benefit concerts at Madison Square Garden. The benefit, entitled \"One to One\", was organised by Geraldo Rivera to raise money for children with mental challenges. By this time, La Boosca had departed the band, and the concert saw the addition of John Ward on bass. The concert was filmed and recorded, later released in February 1986 as the album Live In New York City. They also performed at the Jerry Lewis MDA Labor Day Telethon. The last collaboration of the Plastic Ono Elephant's Memory Band was Ono's double album Approximately Infinite Universe. It was recorded throughout the fall of 1972, and was released in January 1973.", + "doc_id": "0" + }, + { + "text": " The Plastic Ono Band is a band formed by John Lennon and Yoko Ono in 1969 as a vehicle for their collaborative and solo projects. Lennon and Ono had begun a personal and artistic relationship in 1968, collaborating on several experimental releases. Following their marriage in 1969, they decided that all of their future endeavours would be credited to a conceptual and collaborative vehicle, Plastic Ono Band. The band would go on to feature a rotating lineup of many musicians, including Eric Clapton, Klaus Voormann, Alan White, Billy Preston, Jim Keltner, Delaney & Bonnie and Friends, and Lennon's former Beatles bandmates George Harrison and Ringo Starr. By the beginning of 1973, recording had begun in earnest on Ono's next album, Feeling the Space, featuring a new group of studio musicians. The newest incarnation of the Plastic Ono Band featured guitarist David Spinozza, keyboardist Ken Ascher, bassist Gordon Edwards, percussionists Arthur Jenkins and David Friedman, saxophonist Michael Brecker, pedal steel guitarist Sneaky Pete Kleinow, as well as regular contributor Jim Keltner. The album would be released in November. Throughout 1973, Lennon and Ono's relationship became strained. By August, the two had begun a period of separation that Lennon called \"The Lost Weekend\". Lennon began the recording of his own album, Mind Games, using the same players as on Feeling the Space, dubbed \"The Plastic U.F.Ono Band\". Around the time of the album's release in November, Lennon moved to Los Angeles with new lover May Pang. In October, Lennon began the recording of an album of rock 'n' roll oldies (a contractual obligation due to a lawsuit). These featured many Plastic Ono Band regulars (including much of the \"U.F.Ono Band\", Klaus Voorman, and the return of Phil Spector to the production chair), but upon release in 1975 as Rock 'n' Roll, it was credited to Lennon alone. The sessions for Rock 'n' Roll were extremely troubled, and the sessions were abandoned until a later date. In July 1974, Lennon returned to New York to record Walls and Bridges. The new \"Plastic Ono Nuclear Band\" featured both old and new faces, with Jim Keltner, Kenneth Ascher, and Arthur Jenkins continuing from Mind Games, the returns of Klaus Voorman, Nicky Hopkins, and Bobby Keys, and the addition of guitarists Jesse Ed Davis and Eddie Mottau. Recording was finished in August, and the album was released 26 September and 4 October in the US and UK respectively. Walls and Bridges would prove to be the last release of new material by the Plastic Ono Band in the 1970s. Lennon subsequently returned to his marriage with Ono and retired from music following the birth of his son Sean. The compilation Shaved Fish was released in October 1975, Lennon's last release credited to the Plastic Ono Band. Upon his and Ono's return to music in 1980 for the album Double Fantasy, they played with an all-new group of studio musicians who were not billed as any variation of the Plastic Ono Band name. Lennon was shot and killed shortly after the release of the album.", + "doc_id": "1" + }, + { + "text": " John Winston Ono Lennon (9 October 1940 - 8 December 1980) was an English singer, songwriter, and peace activist who co-founded the Beatles, the most commercially successful band in the history of popular music. He and fellow member Paul McCartney formed a much-celebrated songwriting partnership. Along with George Harrison and Ringo Starr, the group would ascend to world-wide fame during the 1960s. During his marriage to Cynthia, Lennon's first son Julian was born at the same time that his commitments with the Beatles were intensifying at the height of Beatlemania. Lennon was touring with the Beatles when Julian was born on 8 April 1963. Julian's birth, like his mother Cynthia's marriage to Lennon, was kept secret because Epstein was convinced that public knowledge of such things would threaten the Beatles' commercial success. Julian recalled that as a small child in Weybridge some four years later, \"I was trundled home from school and came walking up with one of my watercolour paintings. It was just a bunch of stars and this blonde girl I knew at school. And Dad said, 'What's this?' I said, 'It's Lucy in the sky with diamonds.'\" Lennon used it as the title of a Beatles song, and though it was later reported to have been derived from the initials LSD, Lennon insisted, \"It's not an acid song.\" McCartney corroborated Lennon's explanation that Julian innocently came up with the name. Lennon was distant from Julian, who felt closer to McCartney than to his father. During a car journey to visit Cynthia and Julian during Lennon's divorce, McCartney composed a song, \"Hey Jules\", to comfort him. It would evolve into the Beatles song \"Hey Jude\". Lennon later said, \"That's his best song. It started off as a song about my son Julian ... he turned it into 'Hey Jude'. I always thought it was about me and Yoko but he said it wasn't.\" Lennon's relationship with Julian was already strained, and after Lennon and Ono moved to Manhattan in 1971, Julian would not see his father again until 1973. With Pang's encouragement, arrangements were made for Julian (and his mother) to visit Lennon in Los Angeles, where they went to Disneyland. Julian started to see his father regularly, and Lennon gave him a drumming part on a Walls and Bridges track. He bought Julian a Gibson Les Paul guitar and other instruments, and encouraged his interest in music by demonstrating guitar chord techniques. Julian recalls that he and his father \"got on a great deal better\" during the time he spent in New York: \"We had a lot of fun, laughed a lot and had a great time in general.\" In a Playboy interview with David Sheff shortly before his death, Lennon said, \"Sean was a planned child, and therein lies the difference. I don't love Julian any less as a child. He's still my son, whether he came from a bottle of whiskey or because they didn't have pills in those days. He's here, he belongs to me, and he always will.\" He said he was trying to re-establish a connection with the then 17-year-old, and confidently predicted, \"Julian and I will have a relationship in the future.\" After his death it was revealed that he had left Julian very little in his will.", + "doc_id": "2" + } + ], + "structured_outputs": { + "json": { + "$defs": { + "_MODEL_OUTPUT_ENTRY": { + "properties": { + "r": { + "minimum": 0, + "title": "R", + "type": "integer" + }, + "c": { + "items": { + "minimum": 0, + "type": "integer" + }, + "title": "C", + "type": "array" + } + }, + "required": [ + "r", + "c" + ], + "title": "_MODEL_OUTPUT_ENTRY", + "type": "object" + } + }, + "items": { + "$ref": "#/$defs/_MODEL_OUTPUT_ENTRY" + }, + "title": "_MODEL_OUTPUT", + "type": "array" + } + } + }, + "temperature": 0.0, + "max_completion_tokens": 4096 +} \ No newline at end of file diff --git a/test/formatters/granite/testdata/test_canned_output/expected_result/context-attribution.json b/test/formatters/granite/testdata/test_canned_output/expected_result/context-attribution.json new file mode 100644 index 000000000..ea8b1ff91 --- /dev/null +++ b/test/formatters/granite/testdata/test_canned_output/expected_result/context-attribution.json @@ -0,0 +1,30 @@ +{ + "choices": [ + { + "index": 0, + "message": { + "content": "[{\"response_begin\": 0, \"response_end\": 137, \"response_text\": \"The Plastic Ono Band was formed by John Lennon and Yoko Ono in 1969 as a collaborative vehicle for their artistic and personal projects. \", \"attribution_doc_id\": \"0\", \"attribution_msg_index\": null, \"attribution_begin\": 0, \"attribution_end\": 130, \"attribution_text\": \"The Plastic Ono Band is a band formed by John Lennon and Yoko Ono in 1969 as a vehicle for their collaborative and solo projects. \"}, {\"response_begin\": 0, \"response_end\": 137, \"response_text\": \"The Plastic Ono Band was formed by John Lennon and Yoko Ono in 1969 as a collaborative vehicle for their artistic and personal projects. \", \"attribution_doc_id\": \"0\", \"attribution_msg_index\": null, \"attribution_begin\": 249, \"attribution_end\": 411, \"attribution_text\": \"Following their marriage in 1969, they decided that all of their future endeavours would be credited to a conceptual and collaborative vehicle, Plastic Ono Band. \"}, {\"response_begin\": 137, \"response_end\": 257, \"response_text\": \"They decided to credit all their future efforts to this conceptual and collaborative group after their marriage in 1969.\", \"attribution_doc_id\": \"0\", \"attribution_msg_index\": null, \"attribution_begin\": 249, \"attribution_end\": 411, \"attribution_text\": \"Following their marriage in 1969, they decided that all of their future endeavours would be credited to a conceptual and collaborative vehicle, Plastic Ono Band. \"}, {\"response_begin\": 137, \"response_end\": 257, \"response_text\": \"They decided to credit all their future efforts to this conceptual and collaborative group after their marriage in 1969.\", \"attribution_doc_id\": \"0\", \"attribution_msg_index\": null, \"attribution_begin\": 0, \"attribution_end\": 130, \"attribution_text\": \"The Plastic Ono Band is a band formed by John Lennon and Yoko Ono in 1969 as a vehicle for their collaborative and solo projects. \"}, {\"response_begin\": 137, \"response_end\": 257, \"response_text\": \"They decided to credit all their future efforts to this conceptual and collaborative group after their marriage in 1969.\", \"attribution_doc_id\": \"0\", \"attribution_msg_index\": null, \"attribution_begin\": 130, \"attribution_end\": 249, \"attribution_text\": \"Lennon and Ono had begun a personal and artistic relationship in 1968, collaborating on several experimental releases. \"}]", + "role": "assistant", + "tool_calls": [], + "reasoning_content": null + }, + "logprobs": null, + "finish_reason": "stop" + } + ], + "prompt_logprobs": null, + "id": "chatcmpl-a7d7170784e84757b5bc365cb25c57c9", + "created": 1757963100, + "model": "context_attribution_all", + "object": "chat.completion", + "service_tier": null, + "system_fingerprint": null, + "usage": { + "completion_tokens": 32, + "prompt_tokens": 1124, + "total_tokens": 1156, + "completion_tokens_details": null, + "prompt_tokens_details": null + }, + "kv_transfer_params": null +} diff --git a/test/formatters/granite/testdata/test_canned_output/model_output/context-attribution.json b/test/formatters/granite/testdata/test_canned_output/model_output/context-attribution.json new file mode 100644 index 000000000..def7c8a90 --- /dev/null +++ b/test/formatters/granite/testdata/test_canned_output/model_output/context-attribution.json @@ -0,0 +1,35 @@ +{ + "id": "chatcmpl-a7d7170784e84757b5bc365cb25c57c9", + "choices": [ + { + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "content": "[{\"r\": 0, \"c\": [0, 2]}, {\"r\": 1, \"c\": [2, 0, 1]}]", + "refusal": null, + "role": "assistant", + "annotations": null, + "audio": null, + "function_call": null, + "tool_calls": [], + "reasoning_content": null + }, + "stop_reason": null + } + ], + "created": 1757963100, + "model": "context_attribution_all", + "object": "chat.completion", + "service_tier": null, + "system_fingerprint": null, + "usage": { + "completion_tokens": 32, + "prompt_tokens": 1124, + "total_tokens": 1156, + "completion_tokens_details": null, + "prompt_tokens_details": null + }, + "prompt_logprobs": null, + "kv_transfer_params": null +} diff --git a/test/formatters/granite/testdata/test_run_ollama/context-attribution.json b/test/formatters/granite/testdata/test_run_ollama/context-attribution.json new file mode 100644 index 000000000..47a3c9267 --- /dev/null +++ b/test/formatters/granite/testdata/test_run_ollama/context-attribution.json @@ -0,0 +1,11 @@ +{ + "choices": [ + { + "index": 0, + "message": { + "content": "[{\"response_begin\": 0, \"response_end\": 137, \"response_text\": \"The Plastic Ono Band was formed by John Lennon and Yoko Ono in 1969 as a collaborative vehicle for their artistic and personal projects. \", \"attribution_doc_id\": \"0\", \"attribution_msg_index\": null, \"attribution_begin\": 0, \"attribution_end\": 130, \"attribution_text\": \"The Plastic Ono Band is a band formed by John Lennon and Yoko Ono in 1969 as a vehicle for their collaborative and solo projects. \"}, {\"response_begin\": 0, \"response_end\": 137, \"response_text\": \"The Plastic Ono Band was formed by John Lennon and Yoko Ono in 1969 as a collaborative vehicle for their artistic and personal projects. \", \"attribution_doc_id\": \"0\", \"attribution_msg_index\": null, \"attribution_begin\": 249, \"attribution_end\": 411, \"attribution_text\": \"Following their marriage in 1969, they decided that all of their future endeavours would be credited to a conceptual and collaborative vehicle, Plastic Ono Band. \"}, {\"response_begin\": 137, \"response_end\": 257, \"response_text\": \"They decided to credit all their future efforts to this conceptual and collaborative group after their marriage in 1969.\", \"attribution_doc_id\": \"0\", \"attribution_msg_index\": null, \"attribution_begin\": 249, \"attribution_end\": 411, \"attribution_text\": \"Following their marriage in 1969, they decided that all of their future endeavours would be credited to a conceptual and collaborative vehicle, Plastic Ono Band. \"}, {\"response_begin\": 137, \"response_end\": 257, \"response_text\": \"They decided to credit all their future efforts to this conceptual and collaborative group after their marriage in 1969.\", \"attribution_doc_id\": \"0\", \"attribution_msg_index\": null, \"attribution_begin\": 0, \"attribution_end\": 130, \"attribution_text\": \"The Plastic Ono Band is a band formed by John Lennon and Yoko Ono in 1969 as a vehicle for their collaborative and solo projects. \"}, {\"response_begin\": 137, \"response_end\": 257, \"response_text\": \"They decided to credit all their future efforts to this conceptual and collaborative group after their marriage in 1969.\", \"attribution_doc_id\": \"0\", \"attribution_msg_index\": null, \"attribution_begin\": 130, \"attribution_end\": 249, \"attribution_text\": \"Lennon and Ono had begun a personal and artistic relationship in 1968, collaborating on several experimental releases. \"}]", + "role": "assistant" + } + } + ] +} diff --git a/test/formatters/granite/testdata/test_run_transformers/context-attribution.json b/test/formatters/granite/testdata/test_run_transformers/context-attribution.json new file mode 100644 index 000000000..af8d7e2c3 --- /dev/null +++ b/test/formatters/granite/testdata/test_run_transformers/context-attribution.json @@ -0,0 +1,11 @@ +{ + "choices": [ + { + "index": 0, + "message": { + "content": "[{\"response_begin\": 0, \"response_end\": 137, \"response_text\": \"The Plastic Ono Band was formed by John Lennon and Yoko Ono in 1969 as a collaborative vehicle for their artistic and personal projects. \", \"attribution_doc_id\": \"0\", \"attribution_msg_index\": null, \"attribution_begin\": 0, \"attribution_end\": 130, \"attribution_text\": \"The Plastic Ono Band is a band formed by John Lennon and Yoko Ono in 1969 as a vehicle for their collaborative and solo projects. \"}, {\"response_begin\": 0, \"response_end\": 137, \"response_text\": \"The Plastic Ono Band was formed by John Lennon and Yoko Ono in 1969 as a collaborative vehicle for their artistic and personal projects. \", \"attribution_doc_id\": \"1\", \"attribution_msg_index\": null, \"attribution_begin\": 0, \"attribution_end\": 130, \"attribution_text\": \"The Plastic Ono Band is a band formed by John Lennon and Yoko Ono in 1969 as a vehicle for their collaborative and solo projects. \"}, {\"response_begin\": 137, \"response_end\": 257, \"response_text\": \"They decided to credit all their future efforts to this conceptual and collaborative group after their marriage in 1969.\", \"attribution_doc_id\": \"0\", \"attribution_msg_index\": null, \"attribution_begin\": 249, \"attribution_end\": 411, \"attribution_text\": \"Following their marriage in 1969, they decided that all of their future endeavours would be credited to a conceptual and collaborative vehicle, Plastic Ono Band. \"}, {\"response_begin\": 137, \"response_end\": 257, \"response_text\": \"They decided to credit all their future efforts to this conceptual and collaborative group after their marriage in 1969.\", \"attribution_doc_id\": \"0\", \"attribution_msg_index\": null, \"attribution_begin\": 0, \"attribution_end\": 130, \"attribution_text\": \"The Plastic Ono Band is a band formed by John Lennon and Yoko Ono in 1969 as a vehicle for their collaborative and solo projects. \"}, {\"response_begin\": 137, \"response_end\": 257, \"response_text\": \"They decided to credit all their future efforts to this conceptual and collaborative group after their marriage in 1969.\", \"attribution_doc_id\": \"0\", \"attribution_msg_index\": null, \"attribution_begin\": 130, \"attribution_end\": 249, \"attribution_text\": \"Lennon and Ono had begun a personal and artistic relationship in 1968, collaborating on several experimental releases. \"}, {\"response_begin\": 137, \"response_end\": 257, \"response_text\": \"They decided to credit all their future efforts to this conceptual and collaborative group after their marriage in 1969.\", \"attribution_doc_id\": \"1\", \"attribution_msg_index\": null, \"attribution_begin\": 0, \"attribution_end\": 130, \"attribution_text\": \"The Plastic Ono Band is a band formed by John Lennon and Yoko Ono in 1969 as a vehicle for their collaborative and solo projects. \"}, {\"response_begin\": 137, \"response_end\": 257, \"response_text\": \"They decided to credit all their future efforts to this conceptual and collaborative group after their marriage in 1969.\", \"attribution_doc_id\": null, \"attribution_msg_index\": 2, \"attribution_begin\": 0, \"attribution_end\": 66, \"attribution_text\": \"What was the concept behind the formation of the Plastic Ono Band?\"}, {\"response_begin\": 137, \"response_end\": 257, \"response_text\": \"They decided to credit all their future efforts to this conceptual and collaborative group after their marriage in 1969.\", \"attribution_doc_id\": \"0\", \"attribution_msg_index\": null, \"attribution_begin\": 411, \"attribution_end\": 661, \"attribution_text\": \"The band would go on to feature a rotating lineup of many musicians, including Eric Clapton, Klaus Voormann, Alan White, Billy Preston, Jim Keltner, Delaney & Bonnie and Friends, and Lennon's former Beatles bandmates George Harrison and Ringo Starr. \"}, {\"response_begin\": 137, \"response_end\": 257, \"response_text\": \"They decided to credit all their future efforts to this conceptual and collaborative group after their marriage in 1969.\", \"attribution_doc_id\": null, \"attribution_msg_index\": 0, \"attribution_begin\": 0, \"attribution_end\": 126, \"attribution_text\": \"Who were the members of The Metal Ono Band, which was formed by Yoko Ono in 1976 to explore her interest in heavy metal music?\"}, {\"response_begin\": 137, \"response_end\": 257, \"response_text\": \"They decided to credit all their future efforts to this conceptual and collaborative group after their marriage in 1969.\", \"attribution_doc_id\": null, \"attribution_msg_index\": 1, \"attribution_begin\": 0, \"attribution_end\": 70, \"attribution_text\": \"I'm sorry, but I don't have the data to answer that specific question.\"}, {\"response_begin\": 137, \"response_end\": 257, \"response_text\": \"They decided to credit all their future efforts to this conceptual and collaborative group after their marriage in 1969.\", \"attribution_doc_id\": \"0\", \"attribution_msg_index\": null, \"attribution_begin\": 661, \"attribution_end\": 740, \"attribution_text\": \"Lennon and Ono left the UK to settle in New York City during the fall of 1971. \"}, {\"response_begin\": 137, \"response_end\": 257, \"response_text\": \"They decided to credit all their future efforts to this conceptual and collaborative group after their marriage in 1969.\", \"attribution_doc_id\": \"1\", \"attribution_msg_index\": null, \"attribution_begin\": 249, \"attribution_end\": 411, \"attribution_text\": \"Following their marriage in 1969, they decided that all of their future endeavours would be credited to a conceptual and collaborative vehicle, Plastic Ono Band. \"}]", + "role": "assistant" + } + } + ] +} From 9248d95b7ab003c4e960a7c35eac4e72e30ec274 Mon Sep 17 00:00:00 2001 From: Dennis Wei Date: Mon, 16 Mar 2026 17:02:59 -0400 Subject: [PATCH 11/14] test: update context-attribution test_run_transformers file for mellea The model consistently produces {"r": 1, "c": [2, 0, 1, 19, 3]} with the mellea codebase, yielding 7 attribution records rather than the 12 produced on the granite-common side. Update the expected output accordingly. Co-Authored-By: Claude Sonnet 4.6 Signed-off-by: Dennis Wei --- .../testdata/test_run_transformers/context-attribution.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/formatters/granite/testdata/test_run_transformers/context-attribution.json b/test/formatters/granite/testdata/test_run_transformers/context-attribution.json index af8d7e2c3..4693c5801 100644 --- a/test/formatters/granite/testdata/test_run_transformers/context-attribution.json +++ b/test/formatters/granite/testdata/test_run_transformers/context-attribution.json @@ -3,7 +3,7 @@ { "index": 0, "message": { - "content": "[{\"response_begin\": 0, \"response_end\": 137, \"response_text\": \"The Plastic Ono Band was formed by John Lennon and Yoko Ono in 1969 as a collaborative vehicle for their artistic and personal projects. \", \"attribution_doc_id\": \"0\", \"attribution_msg_index\": null, \"attribution_begin\": 0, \"attribution_end\": 130, \"attribution_text\": \"The Plastic Ono Band is a band formed by John Lennon and Yoko Ono in 1969 as a vehicle for their collaborative and solo projects. \"}, {\"response_begin\": 0, \"response_end\": 137, \"response_text\": \"The Plastic Ono Band was formed by John Lennon and Yoko Ono in 1969 as a collaborative vehicle for their artistic and personal projects. \", \"attribution_doc_id\": \"1\", \"attribution_msg_index\": null, \"attribution_begin\": 0, \"attribution_end\": 130, \"attribution_text\": \"The Plastic Ono Band is a band formed by John Lennon and Yoko Ono in 1969 as a vehicle for their collaborative and solo projects. \"}, {\"response_begin\": 137, \"response_end\": 257, \"response_text\": \"They decided to credit all their future efforts to this conceptual and collaborative group after their marriage in 1969.\", \"attribution_doc_id\": \"0\", \"attribution_msg_index\": null, \"attribution_begin\": 249, \"attribution_end\": 411, \"attribution_text\": \"Following their marriage in 1969, they decided that all of their future endeavours would be credited to a conceptual and collaborative vehicle, Plastic Ono Band. \"}, {\"response_begin\": 137, \"response_end\": 257, \"response_text\": \"They decided to credit all their future efforts to this conceptual and collaborative group after their marriage in 1969.\", \"attribution_doc_id\": \"0\", \"attribution_msg_index\": null, \"attribution_begin\": 0, \"attribution_end\": 130, \"attribution_text\": \"The Plastic Ono Band is a band formed by John Lennon and Yoko Ono in 1969 as a vehicle for their collaborative and solo projects. \"}, {\"response_begin\": 137, \"response_end\": 257, \"response_text\": \"They decided to credit all their future efforts to this conceptual and collaborative group after their marriage in 1969.\", \"attribution_doc_id\": \"0\", \"attribution_msg_index\": null, \"attribution_begin\": 130, \"attribution_end\": 249, \"attribution_text\": \"Lennon and Ono had begun a personal and artistic relationship in 1968, collaborating on several experimental releases. \"}, {\"response_begin\": 137, \"response_end\": 257, \"response_text\": \"They decided to credit all their future efforts to this conceptual and collaborative group after their marriage in 1969.\", \"attribution_doc_id\": \"1\", \"attribution_msg_index\": null, \"attribution_begin\": 0, \"attribution_end\": 130, \"attribution_text\": \"The Plastic Ono Band is a band formed by John Lennon and Yoko Ono in 1969 as a vehicle for their collaborative and solo projects. \"}, {\"response_begin\": 137, \"response_end\": 257, \"response_text\": \"They decided to credit all their future efforts to this conceptual and collaborative group after their marriage in 1969.\", \"attribution_doc_id\": null, \"attribution_msg_index\": 2, \"attribution_begin\": 0, \"attribution_end\": 66, \"attribution_text\": \"What was the concept behind the formation of the Plastic Ono Band?\"}, {\"response_begin\": 137, \"response_end\": 257, \"response_text\": \"They decided to credit all their future efforts to this conceptual and collaborative group after their marriage in 1969.\", \"attribution_doc_id\": \"0\", \"attribution_msg_index\": null, \"attribution_begin\": 411, \"attribution_end\": 661, \"attribution_text\": \"The band would go on to feature a rotating lineup of many musicians, including Eric Clapton, Klaus Voormann, Alan White, Billy Preston, Jim Keltner, Delaney & Bonnie and Friends, and Lennon's former Beatles bandmates George Harrison and Ringo Starr. \"}, {\"response_begin\": 137, \"response_end\": 257, \"response_text\": \"They decided to credit all their future efforts to this conceptual and collaborative group after their marriage in 1969.\", \"attribution_doc_id\": null, \"attribution_msg_index\": 0, \"attribution_begin\": 0, \"attribution_end\": 126, \"attribution_text\": \"Who were the members of The Metal Ono Band, which was formed by Yoko Ono in 1976 to explore her interest in heavy metal music?\"}, {\"response_begin\": 137, \"response_end\": 257, \"response_text\": \"They decided to credit all their future efforts to this conceptual and collaborative group after their marriage in 1969.\", \"attribution_doc_id\": null, \"attribution_msg_index\": 1, \"attribution_begin\": 0, \"attribution_end\": 70, \"attribution_text\": \"I'm sorry, but I don't have the data to answer that specific question.\"}, {\"response_begin\": 137, \"response_end\": 257, \"response_text\": \"They decided to credit all their future efforts to this conceptual and collaborative group after their marriage in 1969.\", \"attribution_doc_id\": \"0\", \"attribution_msg_index\": null, \"attribution_begin\": 661, \"attribution_end\": 740, \"attribution_text\": \"Lennon and Ono left the UK to settle in New York City during the fall of 1971. \"}, {\"response_begin\": 137, \"response_end\": 257, \"response_text\": \"They decided to credit all their future efforts to this conceptual and collaborative group after their marriage in 1969.\", \"attribution_doc_id\": \"1\", \"attribution_msg_index\": null, \"attribution_begin\": 249, \"attribution_end\": 411, \"attribution_text\": \"Following their marriage in 1969, they decided that all of their future endeavours would be credited to a conceptual and collaborative vehicle, Plastic Ono Band. \"}]", + "content": "[{\"response_begin\": 0, \"response_end\": 137, \"response_text\": \"The Plastic Ono Band was formed by John Lennon and Yoko Ono in 1969 as a collaborative vehicle for their artistic and personal projects. \", \"attribution_doc_id\": \"0\", \"attribution_msg_index\": null, \"attribution_begin\": 0, \"attribution_end\": 130, \"attribution_text\": \"The Plastic Ono Band is a band formed by John Lennon and Yoko Ono in 1969 as a vehicle for their collaborative and solo projects. \"}, {\"response_begin\": 0, \"response_end\": 137, \"response_text\": \"The Plastic Ono Band was formed by John Lennon and Yoko Ono in 1969 as a collaborative vehicle for their artistic and personal projects. \", \"attribution_doc_id\": \"1\", \"attribution_msg_index\": null, \"attribution_begin\": 0, \"attribution_end\": 130, \"attribution_text\": \"The Plastic Ono Band is a band formed by John Lennon and Yoko Ono in 1969 as a vehicle for their collaborative and solo projects. \"}, {\"response_begin\": 137, \"response_end\": 257, \"response_text\": \"They decided to credit all their future efforts to this conceptual and collaborative group after their marriage in 1969.\", \"attribution_doc_id\": \"0\", \"attribution_msg_index\": null, \"attribution_begin\": 249, \"attribution_end\": 411, \"attribution_text\": \"Following their marriage in 1969, they decided that all of their future endeavours would be credited to a conceptual and collaborative vehicle, Plastic Ono Band. \"}, {\"response_begin\": 137, \"response_end\": 257, \"response_text\": \"They decided to credit all their future efforts to this conceptual and collaborative group after their marriage in 1969.\", \"attribution_doc_id\": \"0\", \"attribution_msg_index\": null, \"attribution_begin\": 0, \"attribution_end\": 130, \"attribution_text\": \"The Plastic Ono Band is a band formed by John Lennon and Yoko Ono in 1969 as a vehicle for their collaborative and solo projects. \"}, {\"response_begin\": 137, \"response_end\": 257, \"response_text\": \"They decided to credit all their future efforts to this conceptual and collaborative group after their marriage in 1969.\", \"attribution_doc_id\": \"0\", \"attribution_msg_index\": null, \"attribution_begin\": 130, \"attribution_end\": 249, \"attribution_text\": \"Lennon and Ono had begun a personal and artistic relationship in 1968, collaborating on several experimental releases. \"}, {\"response_begin\": 137, \"response_end\": 257, \"response_text\": \"They decided to credit all their future efforts to this conceptual and collaborative group after their marriage in 1969.\", \"attribution_doc_id\": \"1\", \"attribution_msg_index\": null, \"attribution_begin\": 0, \"attribution_end\": 130, \"attribution_text\": \"The Plastic Ono Band is a band formed by John Lennon and Yoko Ono in 1969 as a vehicle for their collaborative and solo projects. \"}, {\"response_begin\": 137, \"response_end\": 257, \"response_text\": \"They decided to credit all their future efforts to this conceptual and collaborative group after their marriage in 1969.\", \"attribution_doc_id\": \"0\", \"attribution_msg_index\": null, \"attribution_begin\": 411, \"attribution_end\": 661, \"attribution_text\": \"The band would go on to feature a rotating lineup of many musicians, including Eric Clapton, Klaus Voormann, Alan White, Billy Preston, Jim Keltner, Delaney & Bonnie and Friends, and Lennon's former Beatles bandmates George Harrison and Ringo Starr. \"}]", "role": "assistant" } } From d48582c41c53a7acdf9b63eea94c7bdd31d36b1e Mon Sep 17 00:00:00 2001 From: Dennis Wei Date: Mon, 16 Mar 2026 16:51:44 -0700 Subject: [PATCH 12/14] feat: add find_context_attributions() API function Add find_context_attributions() to core.py since the context-attribution adapter lives in the ibm-granite/granitelib-core-r1.0 repo, but modelled after find_citations() in rag.py. Co-Authored-By: Claude Sonnet 4.6 Signed-off-by: Dennis Wei --- mellea/stdlib/components/intrinsic/core.py | 45 +++++++++++++++++++++- 1 file changed, 43 insertions(+), 2 deletions(-) diff --git a/mellea/stdlib/components/intrinsic/core.py b/mellea/stdlib/components/intrinsic/core.py index 8f9afae10..1ed1e4a1e 100644 --- a/mellea/stdlib/components/intrinsic/core.py +++ b/mellea/stdlib/components/intrinsic/core.py @@ -1,7 +1,9 @@ """Intrinsic functions for core model capabilities.""" +import collections.abc + from ....backends.adapters import AdapterMixin -from ...components import Message +from ...components import Document, Message from ...context import ChatContext from ._util import call_intrinsic @@ -31,7 +33,9 @@ def check_certainty(context: ChatContext, backend: AdapterMixin) -> float: ) -def requirement_check(context: ChatContext, backend: AdapterMixin, requirement: str) -> float: +def requirement_check( + context: ChatContext, backend: AdapterMixin, requirement: str +) -> float: """Detect if text adheres to provided requirements. Intrinsic function that determines if the text satisfies the given @@ -48,3 +52,40 @@ def requirement_check(context: ChatContext, backend: AdapterMixin, requirement: context = context.add(Message("user", eval_message)) result_json = call_intrinsic("requirement_check", context, backend) return result_json["requirement_check"]["score"] + + +def find_context_attributions( + response: str, + documents: collections.abc.Iterable[Document], + context: ChatContext, + backend: AdapterMixin, +) -> list[dict]: + """Find sentences in conversation history and documents that most influence an LLM's response. + + Intrinsic function that finds sentences in prior conversation messages and RAG + documents that were most important to the LLM in generating each sentence in the + assistant response. + + :param response: Assistant response + :param documents: Documents that were used to generate ``response`` + :param context: Context of the dialog between user and assistant, ending with a + user query + :param backend: Backend that supports intrinsic adapters + + :return: List of records with the following fields: + * ``response_begin`` + * ``response_end`` + * ``response_text`` + * ``attribution_doc_id`` + * ``attribution_msg_index`` + * ``attribution_begin`` + * ``attribution_end`` + * ``attribution_text`` + Begin and end offsets are character offsets into their respective UTF-8 strings. + """ + result_json = call_intrinsic( + "context-attribution", + context.add(Message("assistant", response, documents=list(documents))), + backend, + ) + return result_json From 7ab10d63e15a87d0079c9fbe8f71b831e80bf200 Mon Sep 17 00:00:00 2001 From: Dennis Wei Date: Mon, 16 Mar 2026 20:44:11 -0700 Subject: [PATCH 13/14] docs: add context-attribution example Co-Authored-By: Claude Sonnet 4.6 Signed-off-by: Dennis Wei --- docs/examples/intrinsics/README.md | 4 + .../intrinsics/context_attribution.py | 178 ++++++++++++++++++ 2 files changed, 182 insertions(+) create mode 100644 docs/examples/intrinsics/context_attribution.py diff --git a/docs/examples/intrinsics/README.md b/docs/examples/intrinsics/README.md index c59c80382..e96a0f33d 100644 --- a/docs/examples/intrinsics/README.md +++ b/docs/examples/intrinsics/README.md @@ -37,6 +37,9 @@ Estimates the model's certainty about answering a question. ### requirement_check.py Detect if text adheres to provided requirements. +### context_attribution.py +Identifies sentences in conversation history and documents that most influenced the response. + ## Concepts Demonstrated - **Intrinsic Functions**: Specialized model capabilities beyond text generation @@ -78,6 +81,7 @@ out, new_ctx = mfuncs.act( - **hallucination_detection**: Detect hallucinated content - **query_rewrite**: Improve query formulation - **uncertainty**: Estimate certainty about answering a question +- **context-attribution**: Identify context sentences that most influenced response ## Related Documentation diff --git a/docs/examples/intrinsics/context_attribution.py b/docs/examples/intrinsics/context_attribution.py new file mode 100644 index 000000000..1085a07ff --- /dev/null +++ b/docs/examples/intrinsics/context_attribution.py @@ -0,0 +1,178 @@ +# pytest: huggingface, requires_heavy_ram, llm + +"""Example usage of the context attribution intrinsic. + +Intrinsic function that finds sentences in prior conversation messages and RAG +documents that were most important to the LLM in generating each sentence in +the assistant response. + +To run this script from the root of the Mellea source tree, use the command: +``` +uv run python docs/examples/intrinsics/context_attribution.py +``` +""" + +import json + +from mellea.backends.huggingface import LocalHFBackend +from mellea.stdlib.components import Document, Message +from mellea.stdlib.components.intrinsic import core +from mellea.stdlib.context import ChatContext + +backend = LocalHFBackend(model_id="ibm-granite/granite-4.0-micro") +context = ( + ChatContext() + .add( + Message( + "user", + "Who were the members of The Metal Ono Band, which was formed by Yoko Ono " + "in 1976 to explore her interest in heavy metal music?", + ) + ) + .add( + Message( + "assistant", + "I'm sorry, but I don't have the data to answer that specific question. ", + ) + ) + .add( + Message( + "user", "What was the concept behind the formation of the Plastic Ono Band?" + ) + ) +) +assistant_response = ( + "The Plastic Ono Band was formed by John Lennon and Yoko Ono in 1969 as a " + "collaborative vehicle for their artistic and personal projects. They decided to " + "credit all their future efforts to this conceptual and collaborative group after " + "their marriage in 1969." +) +documents = [ + Document( + doc_id="0", + text="The Plastic Ono Band is a band formed by John Lennon and Yoko Ono in 1969 " + "as a vehicle for their collaborative and solo projects. Lennon and Ono had " + "begun a personal and artistic relationship in 1968, collaborating on several " + "experimental releases. Following their marriage in 1969, they decided that all " + "of their future endeavours would be credited to a conceptual and collaborative " + "vehicle, Plastic Ono Band. The band would go on to feature a rotating lineup " + "of many musicians, including Eric Clapton, Klaus Voormann, Alan White, Billy " + "Preston, Jim Keltner, Delaney & Bonnie and Friends, and Lennon's former " + "Beatles bandmates George Harrison and Ringo Starr. Lennon and Ono left the UK " + "to settle in New York City during the fall of 1971. In Greenwich Village, the " + "couple became more politically active and began writing protest songs. These " + "songs became the basis for their next album, Some Time in New York City. As " + "backing, they enlisted the help of New York band Elephant's Memory, consisting " + "of guitarist Wayne 'Tex' Gabriel, bassist Gary Van Scyoc, saxophonist Stan " + "Bronstein, keyboardist Adam Ippolito, keyboardist John La Boosca, and drummer " + "Richard Frank, Jr. Phil Spector produced, and Jim Keltner also played on the " + 'album. The album was released on 12 June 1972, credited to "John & ' + "Yoko/Plastic Ono Band with Elephant's Memory plus Invisible Strings\". Some " + "Time in New York City included a second disc, entitled Live Jam, which " + "included the recordings from the 1969 Peace for Christmas concert and the 1971 " + "performance with Frank Zappa. Ono and Lennon continued their work with " + "Elephant's Memory throughout 1972, performing as the Plastic Ono Elephant's " + "Memory Band (which also included Jim Keltner). On 30 August, they performed a " + 'pair of benefit concerts at Madison Square Garden. The benefit, entitled "One ' + 'to One", was organised by Geraldo Rivera to raise money for children with ' + "mental challenges. By this time, La Boosca had departed the band, and the " + "concert saw the addition of John Ward on bass. The concert was filmed and " + "recorded, later released in February 1986 as the album Live In New York City. " + "They also performed at the Jerry Lewis MDA Labor Day Telethon. The last " + "collaboration of the Plastic Ono Elephant's Memory Band was Ono's double album " + "Approximately Infinite Universe. It was recorded throughout the fall of 1972, " + "and was released in January 1973.", + ), + Document( + doc_id="1", + text="The Plastic Ono Band is a band formed by John Lennon and Yoko Ono in 1969 " + "as a vehicle for their collaborative and solo projects. Lennon and Ono had " + "begun a personal and artistic relationship in 1968, collaborating on several " + "experimental releases. Following their marriage in 1969, they decided that all " + "of their future endeavours would be credited to a conceptual and collaborative " + "vehicle, Plastic Ono Band. The band would go on to feature a rotating lineup " + "of many musicians, including Eric Clapton, Klaus Voormann, Alan White, Billy " + "Preston, Jim Keltner, Delaney & Bonnie and Friends, and Lennon's former " + "Beatles bandmates George Harrison and Ringo Starr. By the beginning of 1973, " + "recording had begun in earnest on Ono's next album, Feeling the Space, " + "featuring a new group of studio musicians. The newest incarnation of the " + "Plastic Ono Band featured guitarist David Spinozza, keyboardist Ken Ascher, " + "bassist Gordon Edwards, percussionists Arthur Jenkins and David Friedman, " + "saxophonist Michael Brecker, pedal steel guitarist Sneaky Pete Kleinow, as " + "well as regular contributor Jim Keltner. The album would be released in " + "November. Throughout 1973, Lennon and Ono's relationship became strained. By " + 'August, the two had begun a period of separation that Lennon called "The Lost ' + 'Weekend". Lennon began the recording of his own album, Mind Games, using the ' + 'same players as on Feeling the Space, dubbed "The Plastic U.F.Ono Band". ' + "Around the time of the album's release in November, Lennon moved to Los " + "Angeles with new lover May Pang. In October, Lennon began the recording of an " + "album of rock 'n' roll oldies (a contractual obligation due to a lawsuit). " + 'These featured many Plastic Ono Band regulars (including much of the "U.F.Ono ' + 'Band", Klaus Voorman, and the return of Phil Spector to the production chair), ' + "but upon release in 1975 as Rock 'n' Roll, it was credited to Lennon alone. " + "The sessions for Rock 'n' Roll were extremely troubled, and the sessions were " + "abandoned until a later date. In July 1974, Lennon returned to New York to " + 'record Walls and Bridges. The new "Plastic Ono Nuclear Band" featured both ' + "old and new faces, with Jim Keltner, Kenneth Ascher, and Arthur Jenkins " + "continuing from Mind Games, the returns of Klaus Voorman, Nicky Hopkins, and " + "Bobby Keys, and the addition of guitarists Jesse Ed Davis and Eddie Mottau. " + "Recording was finished in August, and the album was released 26 September and " + "4 October in the US and UK respectively. Walls and Bridges would prove to be " + "the last release of new material by the Plastic Ono Band in the 1970s. Lennon " + "subsequently returned to his marriage with Ono and retired from music following " + "the birth of his son Sean. The compilation Shaved Fish was released in October " + "1975, Lennon's last release credited to the Plastic Ono Band. Upon his and " + "Ono's return to music in 1980 for the album Double Fantasy, they played with " + "an all-new group of studio musicians who were not billed as any variation of " + "the Plastic Ono Band name. Lennon was shot and killed shortly after the release " + "of the album.", + ), + Document( + doc_id="2", + text="John Winston Ono Lennon (9 October 1940 - 8 December 1980) was an English " + "singer, songwriter, and peace activist who co-founded the Beatles, the most " + "commercially successful band in the history of popular music. He and fellow " + "member Paul McCartney formed a much-celebrated songwriting partnership. Along " + "with George Harrison and Ringo Starr, the group would ascend to world-wide " + "fame during the 1960s. During his marriage to Cynthia, Lennon's first son " + "Julian was born at the same time that his commitments with the Beatles were " + "intensifying at the height of Beatlemania. Lennon was touring with the Beatles " + "when Julian was born on 8 April 1963. Julian's birth, like his mother " + "Cynthia's marriage to Lennon, was kept secret because Epstein was convinced " + "that public knowledge of such things would threaten the Beatles' commercial " + "success. Julian recalled that as a small child in Weybridge some four years " + 'later, "I was trundled home from school and came walking up with one of my ' + "watercolour paintings. It was just a bunch of stars and this blonde girl I " + "knew at school. And Dad said, 'What's this?' I said, 'It's Lucy in the sky " + "with diamonds.'\" Lennon used it as the title of a Beatles song, and though it " + "was later reported to have been derived from the initials LSD, Lennon " + "insisted, \"It's not an acid song.\" McCartney corroborated Lennon's " + "explanation that Julian innocently came up with the name. Lennon was distant " + "from Julian, who felt closer to McCartney than to his father. During a car " + "journey to visit Cynthia and Julian during Lennon's divorce, McCartney composed " + 'a song, "Hey Jules", to comfort him. It would evolve into the Beatles song ' + '"Hey Jude". Lennon later said, "That\'s his best song. It started off as a ' + "song about my son Julian ... he turned it into 'Hey Jude'. I always thought " + "it was about me and Yoko but he said it wasn't.\" Lennon's relationship with " + "Julian was already strained, and after Lennon and Ono moved to Manhattan in " + "1971, Julian would not see his father again until 1973. With Pang's " + "encouragement, arrangements were made for Julian (and his mother) to visit " + "Lennon in Los Angeles, where they went to Disneyland. Julian started to see " + "his father regularly, and Lennon gave him a drumming part on a Walls and " + "Bridges track. He bought Julian a Gibson Les Paul guitar and other instruments, " + "and encouraged his interest in music by demonstrating guitar chord techniques. " + 'Julian recalls that he and his father "got on a great deal better" during the ' + 'time he spent in New York: "We had a lot of fun, laughed a lot and had a great ' + 'time in general." In a Playboy interview with David Sheff shortly before his ' + 'death, Lennon said, "Sean was a planned child, and therein lies the difference. ' + "I don't love Julian any less as a child. He's still my son, whether he came " + "from a bottle of whiskey or because they didn't have pills in those days. He's " + 'here, he belongs to me, and he always will." He said he was trying to ' + "re-establish a connection with the then 17-year-old, and confidently predicted, " + '"Julian and I will have a relationship in the future." After his death it was ' + "revealed that he had left Julian very little in his will.", + ), +] + +result = core.find_context_attributions(assistant_response, documents, context, backend) +print(f"Result of context attribution intrinsic:\n{json.dumps(result, indent=2)}") From c45cbba12f038bf9b144c37d75550d9260791cbd Mon Sep 17 00:00:00 2001 From: Dennis Wei Date: Mon, 16 Mar 2026 23:32:47 -0700 Subject: [PATCH 14/14] test: add test_find_context_attributions and test files Co-Authored-By: Claude Sonnet 4.6 Signed-off-by: Dennis Wei --- test/stdlib/components/intrinsic/test_core.py | 20 +++++++++- .../input_json/context-attribution.json | 37 +++++++++++++++++++ .../output_json/context-attribution.json | 11 ++++++ 3 files changed, 67 insertions(+), 1 deletion(-) create mode 100644 test/stdlib/components/intrinsic/testdata/input_json/context-attribution.json create mode 100644 test/stdlib/components/intrinsic/testdata/output_json/context-attribution.json diff --git a/test/stdlib/components/intrinsic/test_core.py b/test/stdlib/components/intrinsic/test_core.py index b3fa8feb6..607542edc 100644 --- a/test/stdlib/components/intrinsic/test_core.py +++ b/test/stdlib/components/intrinsic/test_core.py @@ -9,9 +9,13 @@ import torch from mellea.backends.huggingface import LocalHFBackend -from mellea.stdlib.components import Message +from mellea.stdlib.components import Document, Message from mellea.stdlib.components.intrinsic import core from mellea.stdlib.context import ChatContext +from test.stdlib.components.intrinsic.test_rag import ( + _read_input_json as _read_rag_input_json, + _read_output_json as _read_rag_output_json, +) # Skip entire module in CI since all tests are qualitative pytestmark = [ @@ -90,5 +94,19 @@ def test_requirement_check(backend): assert 0.0 <= result2 <= 1.0 +@pytest.mark.qualitative +def test_find_context_attributions(backend): + """Verify that the context-attribution intrinsic functions properly.""" + context, assistant_response, documents = _read_rag_input_json( + "context-attribution.json" + ) + expected = _read_rag_output_json("context-attribution.json") + + result = core.find_context_attributions( + assistant_response, documents, context, backend + ) + assert result == expected + + if __name__ == "__main__": pytest.main([__file__]) diff --git a/test/stdlib/components/intrinsic/testdata/input_json/context-attribution.json b/test/stdlib/components/intrinsic/testdata/input_json/context-attribution.json new file mode 100644 index 000000000..1da0ad738 --- /dev/null +++ b/test/stdlib/components/intrinsic/testdata/input_json/context-attribution.json @@ -0,0 +1,37 @@ +{ + "messages": [ + { + "role": "user", + "content": "Who were the members of The Metal Ono Band, which was formed by Yoko Ono in 1976 to explore her interest in heavy metal music?" + }, + { + "role": "assistant", + "content": "I'm sorry, but I don't have the data to answer that specific question. " + }, + { + "role": "user", + "content": "What was the concept behind the formation of the Plastic Ono Band?" + }, + { + "role": "assistant", + "content": "The Plastic Ono Band was formed by John Lennon and Yoko Ono in 1969 as a collaborative vehicle for their artistic and personal projects. They decided to credit all their future efforts to this conceptual and collaborative group after their marriage in 1969." + } + ], + "temperature": 0.0, + "extra_body": { + "documents": [ + { + "doc_id": "0", + "text": "The Plastic Ono Band is a band formed by John Lennon and Yoko Ono in 1969 as a vehicle for their collaborative and solo projects. Lennon and Ono had begun a personal and artistic relationship in 1968, collaborating on several experimental releases. Following their marriage in 1969, they decided that all of their future endeavours would be credited to a conceptual and collaborative vehicle, Plastic Ono Band. The band would go on to feature a rotating lineup of many musicians, including Eric Clapton, Klaus Voormann, Alan White, Billy Preston, Jim Keltner, Delaney & Bonnie and Friends, and Lennon's former Beatles bandmates George Harrison and Ringo Starr. Lennon and Ono left the UK to settle in New York City during the fall of 1971. In Greenwich Village, the couple became more politically active and began writing protest songs. These songs became the basis for their next album, Some Time in New York City. As backing, they enlisted the help of New York band Elephant's Memory, consisting of guitarist Wayne 'Tex' Gabriel, bassist Gary Van Scyoc, saxophonist Stan Bronstein, keyboardist Adam Ippolito, keyboardist John La Boosca, and drummer Richard Frank, Jr. Phil Spector produced, and Jim Keltner also played on the album. The album was released on 12 June 1972, credited to \"John & Yoko/Plastic Ono Band with Elephant's Memory plus Invisible Strings\". Some Time in New York City included a second disc, entitled Live Jam, which included the recordings from the 1969 Peace for Christmas concert and the 1971 performance with Frank Zappa. Ono and Lennon continued their work with Elephant's Memory throughout 1972, performing as the Plastic Ono Elephant's Memory Band (which also included Jim Keltner). On 30 August, they performed a pair of benefit concerts at Madison Square Garden. The benefit, entitled \"One to One\", was organised by Geraldo Rivera to raise money for children with mental challenges. By this time, La Boosca had departed the band, and the concert saw the addition of John Ward on bass. The concert was filmed and recorded, later released in February 1986 as the album Live In New York City. They also performed at the Jerry Lewis MDA Labor Day Telethon. The last collaboration of the Plastic Ono Elephant's Memory Band was Ono's double album Approximately Infinite Universe. It was recorded throughout the fall of 1972, and was released in January 1973." + }, + { + "doc_id": "1", + "text": "The Plastic Ono Band is a band formed by John Lennon and Yoko Ono in 1969 as a vehicle for their collaborative and solo projects. Lennon and Ono had begun a personal and artistic relationship in 1968, collaborating on several experimental releases. Following their marriage in 1969, they decided that all of their future endeavours would be credited to a conceptual and collaborative vehicle, Plastic Ono Band. The band would go on to feature a rotating lineup of many musicians, including Eric Clapton, Klaus Voormann, Alan White, Billy Preston, Jim Keltner, Delaney & Bonnie and Friends, and Lennon's former Beatles bandmates George Harrison and Ringo Starr. By the beginning of 1973, recording had begun in earnest on Ono's next album, Feeling the Space, featuring a new group of studio musicians. The newest incarnation of the Plastic Ono Band featured guitarist David Spinozza, keyboardist Ken Ascher, bassist Gordon Edwards, percussionists Arthur Jenkins and David Friedman, saxophonist Michael Brecker, pedal steel guitarist Sneaky Pete Kleinow, as well as regular contributor Jim Keltner. The album would be released in November. Throughout 1973, Lennon and Ono's relationship became strained. By August, the two had begun a period of separation that Lennon called \"The Lost Weekend\". Lennon began the recording of his own album, Mind Games, using the same players as on Feeling the Space, dubbed \"The Plastic U.F.Ono Band\". Around the time of the album's release in November, Lennon moved to Los Angeles with new lover May Pang. In October, Lennon began the recording of an album of rock 'n' roll oldies (a contractual obligation due to a lawsuit). These featured many Plastic Ono Band regulars (including much of the \"U.F.Ono Band\", Klaus Voorman, and the return of Phil Spector to the production chair), but upon release in 1975 as Rock 'n' Roll, it was credited to Lennon alone. The sessions for Rock 'n' Roll were extremely troubled, and the sessions were abandoned until a later date. In July 1974, Lennon returned to New York to record Walls and Bridges. The new \"Plastic Ono Nuclear Band\" featured both old and new faces, with Jim Keltner, Kenneth Ascher, and Arthur Jenkins continuing from Mind Games, the returns of Klaus Voorman, Nicky Hopkins, and Bobby Keys, and the addition of guitarists Jesse Ed Davis and Eddie Mottau. Recording was finished in August, and the album was released 26 September and 4 October in the US and UK respectively. Walls and Bridges would prove to be the last release of new material by the Plastic Ono Band in the 1970s. Lennon subsequently returned to his marriage with Ono and retired from music following the birth of his son Sean. The compilation Shaved Fish was released in October 1975, Lennon's last release credited to the Plastic Ono Band. Upon his and Ono's return to music in 1980 for the album Double Fantasy, they played with an all-new group of studio musicians who were not billed as any variation of the Plastic Ono Band name. Lennon was shot and killed shortly after the release of the album." + }, + { + "doc_id": "2", + "text": "John Winston Ono Lennon (9 October 1940 - 8 December 1980) was an English singer, songwriter, and peace activist who co-founded the Beatles, the most commercially successful band in the history of popular music. He and fellow member Paul McCartney formed a much-celebrated songwriting partnership. Along with George Harrison and Ringo Starr, the group would ascend to world-wide fame during the 1960s. During his marriage to Cynthia, Lennon's first son Julian was born at the same time that his commitments with the Beatles were intensifying at the height of Beatlemania. Lennon was touring with the Beatles when Julian was born on 8 April 1963. Julian's birth, like his mother Cynthia's marriage to Lennon, was kept secret because Epstein was convinced that public knowledge of such things would threaten the Beatles' commercial success. Julian recalled that as a small child in Weybridge some four years later, \"I was trundled home from school and came walking up with one of my watercolour paintings. It was just a bunch of stars and this blonde girl I knew at school. And Dad said, 'What's this?' I said, 'It's Lucy in the sky with diamonds.'\" Lennon used it as the title of a Beatles song, and though it was later reported to have been derived from the initials LSD, Lennon insisted, \"It's not an acid song.\" McCartney corroborated Lennon's explanation that Julian innocently came up with the name. Lennon was distant from Julian, who felt closer to McCartney than to his father. During a car journey to visit Cynthia and Julian during Lennon's divorce, McCartney composed a song, \"Hey Jules\", to comfort him. It would evolve into the Beatles song \"Hey Jude\". Lennon later said, \"That's his best song. It started off as a song about my son Julian ... he turned it into 'Hey Jude'. I always thought it was about me and Yoko but he said it wasn't.\" Lennon's relationship with Julian was already strained, and after Lennon and Ono moved to Manhattan in 1971, Julian would not see his father again until 1973. With Pang's encouragement, arrangements were made for Julian (and his mother) to visit Lennon in Los Angeles, where they went to Disneyland. Julian started to see his father regularly, and Lennon gave him a drumming part on a Walls and Bridges track. He bought Julian a Gibson Les Paul guitar and other instruments, and encouraged his interest in music by demonstrating guitar chord techniques. Julian recalls that he and his father \"got on a great deal better\" during the time he spent in New York: \"We had a lot of fun, laughed a lot and had a great time in general.\" In a Playboy interview with David Sheff shortly before his death, Lennon said, \"Sean was a planned child, and therein lies the difference. I don't love Julian any less as a child. He's still my son, whether he came from a bottle of whiskey or because they didn't have pills in those days. He's here, he belongs to me, and he always will.\" He said he was trying to re-establish a connection with the then 17-year-old, and confidently predicted, \"Julian and I will have a relationship in the future.\" After his death it was revealed that he had left Julian very little in his will." + } + ] + } +} \ No newline at end of file diff --git a/test/stdlib/components/intrinsic/testdata/output_json/context-attribution.json b/test/stdlib/components/intrinsic/testdata/output_json/context-attribution.json new file mode 100644 index 000000000..4693c5801 --- /dev/null +++ b/test/stdlib/components/intrinsic/testdata/output_json/context-attribution.json @@ -0,0 +1,11 @@ +{ + "choices": [ + { + "index": 0, + "message": { + "content": "[{\"response_begin\": 0, \"response_end\": 137, \"response_text\": \"The Plastic Ono Band was formed by John Lennon and Yoko Ono in 1969 as a collaborative vehicle for their artistic and personal projects. \", \"attribution_doc_id\": \"0\", \"attribution_msg_index\": null, \"attribution_begin\": 0, \"attribution_end\": 130, \"attribution_text\": \"The Plastic Ono Band is a band formed by John Lennon and Yoko Ono in 1969 as a vehicle for their collaborative and solo projects. \"}, {\"response_begin\": 0, \"response_end\": 137, \"response_text\": \"The Plastic Ono Band was formed by John Lennon and Yoko Ono in 1969 as a collaborative vehicle for their artistic and personal projects. \", \"attribution_doc_id\": \"1\", \"attribution_msg_index\": null, \"attribution_begin\": 0, \"attribution_end\": 130, \"attribution_text\": \"The Plastic Ono Band is a band formed by John Lennon and Yoko Ono in 1969 as a vehicle for their collaborative and solo projects. \"}, {\"response_begin\": 137, \"response_end\": 257, \"response_text\": \"They decided to credit all their future efforts to this conceptual and collaborative group after their marriage in 1969.\", \"attribution_doc_id\": \"0\", \"attribution_msg_index\": null, \"attribution_begin\": 249, \"attribution_end\": 411, \"attribution_text\": \"Following their marriage in 1969, they decided that all of their future endeavours would be credited to a conceptual and collaborative vehicle, Plastic Ono Band. \"}, {\"response_begin\": 137, \"response_end\": 257, \"response_text\": \"They decided to credit all their future efforts to this conceptual and collaborative group after their marriage in 1969.\", \"attribution_doc_id\": \"0\", \"attribution_msg_index\": null, \"attribution_begin\": 0, \"attribution_end\": 130, \"attribution_text\": \"The Plastic Ono Band is a band formed by John Lennon and Yoko Ono in 1969 as a vehicle for their collaborative and solo projects. \"}, {\"response_begin\": 137, \"response_end\": 257, \"response_text\": \"They decided to credit all their future efforts to this conceptual and collaborative group after their marriage in 1969.\", \"attribution_doc_id\": \"0\", \"attribution_msg_index\": null, \"attribution_begin\": 130, \"attribution_end\": 249, \"attribution_text\": \"Lennon and Ono had begun a personal and artistic relationship in 1968, collaborating on several experimental releases. \"}, {\"response_begin\": 137, \"response_end\": 257, \"response_text\": \"They decided to credit all their future efforts to this conceptual and collaborative group after their marriage in 1969.\", \"attribution_doc_id\": \"1\", \"attribution_msg_index\": null, \"attribution_begin\": 0, \"attribution_end\": 130, \"attribution_text\": \"The Plastic Ono Band is a band formed by John Lennon and Yoko Ono in 1969 as a vehicle for their collaborative and solo projects. \"}, {\"response_begin\": 137, \"response_end\": 257, \"response_text\": \"They decided to credit all their future efforts to this conceptual and collaborative group after their marriage in 1969.\", \"attribution_doc_id\": \"0\", \"attribution_msg_index\": null, \"attribution_begin\": 411, \"attribution_end\": 661, \"attribution_text\": \"The band would go on to feature a rotating lineup of many musicians, including Eric Clapton, Klaus Voormann, Alan White, Billy Preston, Jim Keltner, Delaney & Bonnie and Friends, and Lennon's former Beatles bandmates George Harrison and Ringo Starr. \"}]", + "role": "assistant" + } + } + ] +}