From 1d40dc3c1468053b5535f51a263037d94e085d08 Mon Sep 17 00:00:00 2001 From: Jason Dai Date: Fri, 2 Oct 2026 12:10:36 -0700 Subject: [PATCH] fix: GenAI Client(evals) - Parse JSON-string and array conversation history in evaluate() PiperOrigin-RevId: 992464091 --- .../_genai/_evals_data_converters.py | 41 +++- tests/unit/agentplatform/genai/test_evals.py | 178 ++++++++++++++++++ vertexai/_genai/_evals_data_converters.py | 41 +++- 3 files changed, 256 insertions(+), 4 deletions(-) diff --git a/agentplatform/_genai/_evals_data_converters.py b/agentplatform/_genai/_evals_data_converters.py index 2934ba92ea..a63a482031 100644 --- a/agentplatform/_genai/_evals_data_converters.py +++ b/agentplatform/_genai/_evals_data_converters.py @@ -22,6 +22,8 @@ from google.genai import _common from google.genai import types as genai_types +import numpy as np +import pandas as pd from pydantic import alias_generators from pydantic import ValidationError from typing_extensions import override @@ -194,14 +196,22 @@ def convert(self, raw_data: list[dict[str, Any]]) -> types.EvaluationDataset: % (i, type(item_dict).__name__, item_dict) ) item = copy.deepcopy(item_dict) + for column in ("conversation_history", "history"): + value = item.get(column) + if (isinstance(value, str) and not value.strip()) or ( + pd.api.types.is_scalar(value) and pd.isna(value) + ): + item.pop(column, None) eval_case_id = "eval_case_%s" % i prompt_data = item.pop("prompt", None) if not prompt_data: prompt_data = item.pop("source", None) - conversation_history_data = item.pop("conversation_history", None) + history_column = "conversation_history" + conversation_history_data = item.pop(history_column, None) if conversation_history_data is None: - conversation_history_data = item.pop("history", None) + history_column = "history" + conversation_history_data = item.pop(history_column, None) response_data = item.pop("response", None) reference_data = item.pop("reference", None) system_instruction_data = item.pop("instruction", None) @@ -230,6 +240,33 @@ def convert(self, raw_data: list[dict[str, Any]]) -> types.EvaluationDataset: "Invalid prompt type for case %s: %s" % (i, type(prompt_data)) ) + if isinstance(conversation_history_data, np.ndarray): + conversation_history_data = conversation_history_data.tolist() + elif isinstance(conversation_history_data, tuple): + conversation_history_data = list(conversation_history_data) + elif isinstance(conversation_history_data, str): + try: + conversation_history_data = json.loads(conversation_history_data) + except (ValueError, RecursionError) as e: + logger.warning( + "Could not decode JSON string in '%s' column for case %s:" + " %s. Skipping conversation history.", + history_column, + eval_case_id, + e, + ) + conversation_history_data = None + if conversation_history_data is not None and not isinstance( + conversation_history_data, list + ): + logger.warning( + "Invalid type in '%s' column for case %s. Expected a list or a" + " JSON string of a list, but got %s. Skipping conversation" + " history.", + history_column, + eval_case_id, + type(conversation_history_data), + ) conversation_history: Optional[list[types.evals.Message]] = None if isinstance(conversation_history_data, list): conversation_history = [] diff --git a/tests/unit/agentplatform/genai/test_evals.py b/tests/unit/agentplatform/genai/test_evals.py index 492d9cb429..24516604f0 100644 --- a/tests/unit/agentplatform/genai/test_evals.py +++ b/tests/unit/agentplatform/genai/test_evals.py @@ -18,6 +18,7 @@ import asyncio import enum import importlib +import io import json import os import re @@ -45,9 +46,13 @@ types as agentplatform_genai_types, ) from agentplatform._genai.types import common as common_types +from vertexai._genai import ( + _evals_data_converters as vertexai_evals_data_converters, +) from google.genai import client from google.genai import errors as genai_errors from google.genai import types as genai_types +import numpy as np import pandas as pd import pydantic import pytest @@ -61,6 +66,12 @@ pytestmark = pytest.mark.usefixtures("google_auth_mock") +_CONVERTER_MODULES = pytest.mark.parametrize( + "converters", + [_evals_data_converters, vertexai_evals_data_converters], + ids=["agent_platform", "vertexai"], +) + class TestDropEmptyColumns: """Unit tests for the _drop_empty_columns function.""" @@ -5671,6 +5682,16 @@ def test_convert_with_raw_string_response(self): ) +_GEMINI_HISTORY = [ + {"role": "user", "parts": [{"text": "My code word is BLUE."}]}, + {"role": "model", "parts": [{"text": "Noted."}]}, +] + + +def _csv_round_trip(row: dict[str, object]) -> pd.DataFrame: + return pd.read_csv(io.StringIO(pd.DataFrame([row]).to_csv(index=False))) + + class TestFlattenEvalDataConverter: """Unit tests for the _FlattenEvalDataConverter class.""" @@ -5892,6 +5913,163 @@ def test_convert_with_intermediate_events_as_event_objects(self): == "intermediate event" ) + @_CONVERTER_MODULES + @pytest.mark.parametrize("column", ["conversation_history", "history"]) + def test_convert_json_string_history_from_csv(self, converters, column): + raw_data_df = _csv_round_trip( + { + "prompt": "Code word?", + "response": "BLUE", + column: json.dumps(_GEMINI_HISTORY), + } + ) + + result_dataset = converters._FlattenEvalDataConverter().convert( + raw_data_df.to_dict(orient="records") + ) + + assert [ + message.content + for message in result_dataset.eval_cases[0].conversation_history + ] == [genai_types.Content.model_validate(turn) for turn in _GEMINI_HISTORY] + + @_CONVERTER_MODULES + @pytest.mark.parametrize( + "history", + [ + np.array( + [ + {"role": turn["role"], "parts": np.array(turn["parts"])} + for turn in _GEMINI_HISTORY + ] + ), + tuple(_GEMINI_HISTORY), + ], + ids=["ndarray", "tuple"], + ) + def test_convert_array_history(self, converters, history): + result_dataset = converters._FlattenEvalDataConverter().convert( + [{"prompt": "Code word?", "response": "BLUE", "history": history}] + ) + + assert [ + message.content + for message in result_dataset.eval_cases[0].conversation_history + ] == [genai_types.Content.model_validate(turn) for turn in _GEMINI_HISTORY] + + @_CONVERTER_MODULES + @pytest.mark.parametrize( + "column,value,expected_warning", + [ + ( + "conversation_history", + "[not json", + "Could not decode JSON string in 'conversation_history' column for" + " case eval_case_0", + ), + ( + "history", + json.dumps(_GEMINI_HISTORY[0]), + "Invalid type in 'history' column for case eval_case_0. Expected", + ), + ( + "history", + "[" + "1" * 5000 + "]", + "Could not decode JSON string in 'history' column for case eval_case_0", + ), + ( + "history", + np.array(1.5), + "Invalid type in 'history' column for case eval_case_0. Expected", + ), + ], + ids=["invalid_json", "json_object", "oversized_int", "zero_dim_array"], + ) + def test_convert_invalid_history_value_logs_warning( + self, converters, column, value, expected_warning, caplog + ): + with caplog.at_level("WARNING", logger=converters.logger.name): + result_dataset = converters._FlattenEvalDataConverter().convert( + [{"prompt": "Hello", "response": "Hi", column: value}] + ) + + assert result_dataset.eval_cases[0].conversation_history is None + assert len(caplog.records) == 1 + assert expected_warning in caplog.text + + @_CONVERTER_MODULES + @pytest.mark.parametrize( + "value", + [None, float("nan"), np.float32("nan"), pd.NA, pd.NaT, " "], + ids=["none", "nan", "float32_nan", "pd_na", "pd_nat", "blank"], + ) + def test_convert_empty_history_value_without_warning( + self, converters, value, caplog + ): + with caplog.at_level("WARNING", logger=converters.logger.name): + result_dataset = converters._FlattenEvalDataConverter().convert( + [{"prompt": "Hello", "response": "Hi", "conversation_history": value}] + ) + + assert result_dataset.eval_cases[0].conversation_history is None + assert not caplog.records + + @_CONVERTER_MODULES + def test_convert_missing_conversation_history_falls_back_to_history( + self, converters + ): + raw_data_df = _csv_round_trip( + { + "prompt": "Code word?", + "response": "BLUE", + "conversation_history": None, + "history": json.dumps(_GEMINI_HISTORY), + } + ) + + result_dataset = converters._FlattenEvalDataConverter().convert( + raw_data_df.to_dict(orient="records") + ) + + assert [ + message.content + for message in result_dataset.eval_cases[0].conversation_history + ] == [genai_types.Content.model_validate(turn) for turn in _GEMINI_HISTORY] + + @pytest.mark.usefixtures("mock_eval_dependencies") + def test_evaluate_sends_csv_history_to_multi_turn_metric( + self, mock_api_client_fixture + ): + dataset_df = _csv_round_trip( + { + "prompt": "Code word?", + "response": "BLUE", + "conversation_history": json.dumps(_GEMINI_HISTORY), + } + ) + + with mock.patch.object( + evals.Evals, "_evaluate_instances" + ) as mock_evaluate_instances: + mock_evaluate_instances.return_value = ( + agentplatform_genai_types.EvaluateInstancesResponse( + metric_results=[agentplatform_genai_types.MetricResult(score=1.0)] + ) + ) + evals.Evals(api_client_=mock_api_client_fixture).evaluate( + dataset=dataset_df, + metrics=[ + agentplatform_genai_types.Metric( + name="multi_turn_general_quality_v1" + ) + ], + ) + + instance = mock_evaluate_instances.call_args.kwargs["instance"] + assert [ + content.parts[0].text for content in instance.prompt.contents.contents + ] == ["My code word is BLUE.", "Noted.", "Code word?"] + class TestOpenAIDataConverter: """Unit tests for the _OpenAIDataConverter class.""" diff --git a/vertexai/_genai/_evals_data_converters.py b/vertexai/_genai/_evals_data_converters.py index f293152cbe..fbb33761d7 100644 --- a/vertexai/_genai/_evals_data_converters.py +++ b/vertexai/_genai/_evals_data_converters.py @@ -22,6 +22,8 @@ from google.genai import _common from google.genai import types as genai_types +import numpy as np +import pandas as pd from pydantic import alias_generators from pydantic import ValidationError from typing_extensions import override @@ -194,14 +196,22 @@ def convert(self, raw_data: list[dict[str, Any]]) -> types.EvaluationDataset: % (i, type(item_dict).__name__, item_dict) ) item = copy.deepcopy(item_dict) + for column in ("conversation_history", "history"): + value = item.get(column) + if (isinstance(value, str) and not value.strip()) or ( + pd.api.types.is_scalar(value) and pd.isna(value) + ): + item.pop(column, None) eval_case_id = "eval_case_%s" % i prompt_data = item.pop("prompt", None) if not prompt_data: prompt_data = item.pop("source", None) - conversation_history_data = item.pop("conversation_history", None) + history_column = "conversation_history" + conversation_history_data = item.pop(history_column, None) if conversation_history_data is None: - conversation_history_data = item.pop("history", None) + history_column = "history" + conversation_history_data = item.pop(history_column, None) response_data = item.pop("response", None) reference_data = item.pop("reference", None) system_instruction_data = item.pop("instruction", None) @@ -230,6 +240,33 @@ def convert(self, raw_data: list[dict[str, Any]]) -> types.EvaluationDataset: "Invalid prompt type for case %s: %s" % (i, type(prompt_data)) ) + if isinstance(conversation_history_data, np.ndarray): + conversation_history_data = conversation_history_data.tolist() + elif isinstance(conversation_history_data, tuple): + conversation_history_data = list(conversation_history_data) + elif isinstance(conversation_history_data, str): + try: + conversation_history_data = json.loads(conversation_history_data) + except (ValueError, RecursionError) as e: + logger.warning( + "Could not decode JSON string in '%s' column for case %s:" + " %s. Skipping conversation history.", + history_column, + eval_case_id, + e, + ) + conversation_history_data = None + if conversation_history_data is not None and not isinstance( + conversation_history_data, list + ): + logger.warning( + "Invalid type in '%s' column for case %s. Expected a list or a" + " JSON string of a list, but got %s. Skipping conversation" + " history.", + history_column, + eval_case_id, + type(conversation_history_data), + ) conversation_history: Optional[list[types.evals.Message]] = None if isinstance(conversation_history_data, list): conversation_history = []