Skip to content

fix: GenAI Client(evals) - Parse JSON-string and array conversation history in evaluate() #7181

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
copybara-service wants to merge 1 commit into
base: main
Choose a base branch
from
Open

fix: GenAI Client(evals) - Parse JSON-string and array conversation history in evaluate() #7181

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
41 changes: 39 additions & 2 deletions agentplatform/_genai/_evals_data_converters.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,8 @@

from google.genai import _common
from google.genai import types as genai_types
import numpy as np
import pandas as pd
from pydantic import alias_generators
from pydantic import ValidationError
from typing_extensions import override
Expand Down Expand Up @@ -194,14 +196,22 @@ def convert(self, raw_data: list[dict[str, Any]]) -> types.EvaluationDataset:
% (i, type(item_dict).__name__, item_dict)
)
item = copy.deepcopy(item_dict)
for column in ("conversation_history", "history"):
value = item.get(column)
if (isinstance(value, str) and not value.strip()) or (
pd.api.types.is_scalar(value) and pd.isna(value)
):
item.pop(column, None)
eval_case_id = "eval_case_%s" % i
prompt_data = item.pop("prompt", None)
if not prompt_data:
prompt_data = item.pop("source", None)

conversation_history_data = item.pop("conversation_history", None)
history_column = "conversation_history"
conversation_history_data = item.pop(history_column, None)
if conversation_history_data is None:
conversation_history_data = item.pop("history", None)
history_column = "history"
conversation_history_data = item.pop(history_column, None)
response_data = item.pop("response", None)
reference_data = item.pop("reference", None)
system_instruction_data = item.pop("instruction", None)
Expand Down Expand Up @@ -230,6 +240,33 @@ def convert(self, raw_data: list[dict[str, Any]]) -> types.EvaluationDataset:
"Invalid prompt type for case %s: %s" % (i, type(prompt_data))
)

if isinstance(conversation_history_data, np.ndarray):
conversation_history_data = conversation_history_data.tolist()
elif isinstance(conversation_history_data, tuple):
conversation_history_data = list(conversation_history_data)
elif isinstance(conversation_history_data, str):
try:
conversation_history_data = json.loads(conversation_history_data)
except (ValueError, RecursionError) as e:
logger.warning(
"Could not decode JSON string in '%s' column for case %s:"
" %s. Skipping conversation history.",
history_column,
eval_case_id,
e,
)
conversation_history_data = None
if conversation_history_data is not None and not isinstance(
conversation_history_data, list
):
logger.warning(
"Invalid type in '%s' column for case %s. Expected a list or a"
" JSON string of a list, but got %s. Skipping conversation"
" history.",
history_column,
eval_case_id,
type(conversation_history_data),
)
conversation_history: Optional[list[types.evals.Message]] = None
if isinstance(conversation_history_data, list):
conversation_history = []
Expand Down
178 changes: 178 additions & 0 deletions tests/unit/agentplatform/genai/test_evals.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@
import asyncio
import enum
import importlib
import io
import json
import os
import re
Expand Down Expand Up @@ -45,9 +46,13 @@
types as agentplatform_genai_types,
)
from agentplatform._genai.types import common as common_types
from vertexai._genai import (
_evals_data_converters as vertexai_evals_data_converters,
)
from google.genai import client
from google.genai import errors as genai_errors
from google.genai import types as genai_types
import numpy as np
import pandas as pd
import pydantic
import pytest
Expand All @@ -61,6 +66,12 @@

pytestmark = pytest.mark.usefixtures("google_auth_mock")

_CONVERTER_MODULES = pytest.mark.parametrize(
"converters",
[_evals_data_converters, vertexai_evals_data_converters],
ids=["agent_platform", "vertexai"],
)


class TestDropEmptyColumns:
"""Unit tests for the _drop_empty_columns function."""
Expand Down Expand Up @@ -5671,6 +5682,16 @@ def test_convert_with_raw_string_response(self):
)


_GEMINI_HISTORY = [
{"role": "user", "parts": [{"text": "My code word is BLUE."}]},
{"role": "model", "parts": [{"text": "Noted."}]},
]


def _csv_round_trip(row: dict[str, object]) -> pd.DataFrame:
return pd.read_csv(io.StringIO(pd.DataFrame([row]).to_csv(index=False)))


class TestFlattenEvalDataConverter:
"""Unit tests for the _FlattenEvalDataConverter class."""

Expand Down Expand Up @@ -5892,6 +5913,163 @@ def test_convert_with_intermediate_events_as_event_objects(self):
== "intermediate event"
)

@_CONVERTER_MODULES
@pytest.mark.parametrize("column", ["conversation_history", "history"])
def test_convert_json_string_history_from_csv(self, converters, column):
raw_data_df = _csv_round_trip(
{
"prompt": "Code word?",
"response": "BLUE",
column: json.dumps(_GEMINI_HISTORY),
}
)

result_dataset = converters._FlattenEvalDataConverter().convert(
raw_data_df.to_dict(orient="records")
)

assert [
message.content
for message in result_dataset.eval_cases[0].conversation_history
] == [genai_types.Content.model_validate(turn) for turn in _GEMINI_HISTORY]

@_CONVERTER_MODULES
@pytest.mark.parametrize(
"history",
[
np.array(
[
{"role": turn["role"], "parts": np.array(turn["parts"])}
for turn in _GEMINI_HISTORY
]
),
tuple(_GEMINI_HISTORY),
],
ids=["ndarray", "tuple"],
)
def test_convert_array_history(self, converters, history):
result_dataset = converters._FlattenEvalDataConverter().convert(
[{"prompt": "Code word?", "response": "BLUE", "history": history}]
)

assert [
message.content
for message in result_dataset.eval_cases[0].conversation_history
] == [genai_types.Content.model_validate(turn) for turn in _GEMINI_HISTORY]

@_CONVERTER_MODULES
@pytest.mark.parametrize(
"column,value,expected_warning",
[
(
"conversation_history",
"[not json",
"Could not decode JSON string in 'conversation_history' column for"
" case eval_case_0",
),
(
"history",
json.dumps(_GEMINI_HISTORY[0]),
"Invalid type in 'history' column for case eval_case_0. Expected",
),
(
"history",
"[" + "1" * 5000 + "]",
"Could not decode JSON string in 'history' column for case eval_case_0",
),
(
"history",
np.array(1.5),
"Invalid type in 'history' column for case eval_case_0. Expected",
),
],
ids=["invalid_json", "json_object", "oversized_int", "zero_dim_array"],
)
def test_convert_invalid_history_value_logs_warning(
self, converters, column, value, expected_warning, caplog
):
with caplog.at_level("WARNING", logger=converters.logger.name):
result_dataset = converters._FlattenEvalDataConverter().convert(
[{"prompt": "Hello", "response": "Hi", column: value}]
)

assert result_dataset.eval_cases[0].conversation_history is None
assert len(caplog.records) == 1
assert expected_warning in caplog.text

@_CONVERTER_MODULES
@pytest.mark.parametrize(
"value",
[None, float("nan"), np.float32("nan"), pd.NA, pd.NaT, " "],
ids=["none", "nan", "float32_nan", "pd_na", "pd_nat", "blank"],
)
def test_convert_empty_history_value_without_warning(
self, converters, value, caplog
):
with caplog.at_level("WARNING", logger=converters.logger.name):
result_dataset = converters._FlattenEvalDataConverter().convert(
[{"prompt": "Hello", "response": "Hi", "conversation_history": value}]
)

assert result_dataset.eval_cases[0].conversation_history is None
assert not caplog.records

@_CONVERTER_MODULES
def test_convert_missing_conversation_history_falls_back_to_history(
self, converters
):
raw_data_df = _csv_round_trip(
{
"prompt": "Code word?",
"response": "BLUE",
"conversation_history": None,
"history": json.dumps(_GEMINI_HISTORY),
}
)

result_dataset = converters._FlattenEvalDataConverter().convert(
raw_data_df.to_dict(orient="records")
)

assert [
message.content
for message in result_dataset.eval_cases[0].conversation_history
] == [genai_types.Content.model_validate(turn) for turn in _GEMINI_HISTORY]

@pytest.mark.usefixtures("mock_eval_dependencies")
def test_evaluate_sends_csv_history_to_multi_turn_metric(
self, mock_api_client_fixture
):
dataset_df = _csv_round_trip(
{
"prompt": "Code word?",
"response": "BLUE",
"conversation_history": json.dumps(_GEMINI_HISTORY),
}
)

with mock.patch.object(
evals.Evals, "_evaluate_instances"
) as mock_evaluate_instances:
mock_evaluate_instances.return_value = (
agentplatform_genai_types.EvaluateInstancesResponse(
metric_results=[agentplatform_genai_types.MetricResult(score=1.0)]
)
)
evals.Evals(api_client_=mock_api_client_fixture).evaluate(
dataset=dataset_df,
metrics=[
agentplatform_genai_types.Metric(
name="multi_turn_general_quality_v1"
)
],
)

instance = mock_evaluate_instances.call_args.kwargs["instance"]
assert [
content.parts[0].text for content in instance.prompt.contents.contents
] == ["My code word is BLUE.", "Noted.", "Code word?"]


class TestOpenAIDataConverter:
"""Unit tests for the _OpenAIDataConverter class."""
Expand Down
41 changes: 39 additions & 2 deletions vertexai/_genai/_evals_data_converters.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,8 @@

from google.genai import _common
from google.genai import types as genai_types
import numpy as np
import pandas as pd
from pydantic import alias_generators
from pydantic import ValidationError
from typing_extensions import override
Expand Down Expand Up @@ -194,14 +196,22 @@ def convert(self, raw_data: list[dict[str, Any]]) -> types.EvaluationDataset:
% (i, type(item_dict).__name__, item_dict)
)
item = copy.deepcopy(item_dict)
for column in ("conversation_history", "history"):
value = item.get(column)
if (isinstance(value, str) and not value.strip()) or (
pd.api.types.is_scalar(value) and pd.isna(value)
):
item.pop(column, None)
eval_case_id = "eval_case_%s" % i
prompt_data = item.pop("prompt", None)
if not prompt_data:
prompt_data = item.pop("source", None)

conversation_history_data = item.pop("conversation_history", None)
history_column = "conversation_history"
conversation_history_data = item.pop(history_column, None)
if conversation_history_data is None:
conversation_history_data = item.pop("history", None)
history_column = "history"
conversation_history_data = item.pop(history_column, None)
response_data = item.pop("response", None)
reference_data = item.pop("reference", None)
system_instruction_data = item.pop("instruction", None)
Expand Down Expand Up @@ -230,6 +240,33 @@ def convert(self, raw_data: list[dict[str, Any]]) -> types.EvaluationDataset:
"Invalid prompt type for case %s: %s" % (i, type(prompt_data))
)

if isinstance(conversation_history_data, np.ndarray):
conversation_history_data = conversation_history_data.tolist()
elif isinstance(conversation_history_data, tuple):
conversation_history_data = list(conversation_history_data)
elif isinstance(conversation_history_data, str):
try:
conversation_history_data = json.loads(conversation_history_data)
except (ValueError, RecursionError) as e:
logger.warning(
"Could not decode JSON string in '%s' column for case %s:"
" %s. Skipping conversation history.",
history_column,
eval_case_id,
e,
)
conversation_history_data = None
if conversation_history_data is not None and not isinstance(
conversation_history_data, list
):
logger.warning(
"Invalid type in '%s' column for case %s. Expected a list or a"
" JSON string of a list, but got %s. Skipping conversation"
" history.",
history_column,
eval_case_id,
type(conversation_history_data),
)
conversation_history: Optional[list[types.evals.Message]] = None
if isinstance(conversation_history_data, list):
conversation_history = []
Expand Down
Loading