diff --git a/src/components/EvaluatorPicker.tsx b/src/components/EvaluatorPicker.tsx new file mode 100644 index 000000000..1ff12dae8 --- /dev/null +++ b/src/components/EvaluatorPicker.tsx @@ -0,0 +1,92 @@ +import type { EvaluatorSummary } from "@aws-sdk/client-bedrock-agentcore-control"; +import { useNavigate } from "react-router"; +import type { ScreenProps } from "../handlers/types"; +import { coreOptsFromCtx } from "../handlers/utils"; +import { formatTimestamp } from "./formatTimestamp"; +import { PaginatedTablePicker } from "./PaginatedTablePicker"; +import type { DataTableColumn } from "./ui/data-table"; + +// EvaluatorRow is the flat, display-ready shape the table renders. It also +// satisfies DataTable's `T extends Record` constraint, which the +// SDK's EvaluatorSummary interface does not. +interface EvaluatorRow extends Record { + evaluatorId: string; + evaluatorName: string; + evaluatorType: string; + level: string; + updatedAt: string; +} + +export const evaluatorColumns = [ + { key: "evaluatorName", header: "name", flex: true }, + { key: "evaluatorType", header: "type", width: 12 }, + { key: "level", header: "level", width: 10 }, + { + key: "updatedAt", + header: "updated UTC", + width: 16, + render: formatTimestamp, + }, +] satisfies DataTableColumn[]; + +function toRow(evaluator: EvaluatorSummary): EvaluatorRow { + const id = evaluator.evaluatorId ?? ""; + return { + evaluatorId: id, + evaluatorName: evaluator.evaluatorName ?? id, + evaluatorType: evaluator.evaluatorType ?? "-", + level: evaluator.level ?? "-", + updatedAt: evaluator.updatedAt?.toISOString() ?? "-", + }; +} + +export interface EvaluatorPickerProps extends ScreenProps { + breadcrumb: string[]; + description?: string; + onSelect: (evaluatorId: string) => void; + onEscape?: () => void; +} + +/** + * Fetches the caller's evaluators and renders them as a navigable table. + * + * The shared body of every "pick an evaluator" screen (list, and — in the write + * TUI — update/delete). Esc returns to the parent menu derived from the + * breadcrumb unless a host supplies its own onEscape. + */ +export function EvaluatorPicker({ + ctx, + core, + breadcrumb, + description, + onSelect, + onEscape, +}: EvaluatorPickerProps) { + const opts = coreOptsFromCtx(ctx); + const navigate = useNavigate(); + const goBack = onEscape ?? (() => navigate("/" + breadcrumb.slice(0, -1).join("/"))); + + return ( + { + const response = await core.eval.listEvaluators(token, pageSize, opts); + return { + items: response.evaluators ?? [], + nextToken: response.nextToken, + }; + }} + toRow={toRow} + columns={evaluatorColumns} + getValue={(row) => row.evaluatorId} + onSelect={onSelect} + onBack={goBack} + loadingMessage="Loading evaluators…" + errorMessage={(error) => `Error: ${error.message}`} + emptyMessage="No evaluators found in this Region." + emptyPageMessage="No evaluators on this page." + /> + ); +} diff --git a/src/components/OnlineEvalPicker.tsx b/src/components/OnlineEvalPicker.tsx new file mode 100644 index 000000000..df8141c15 --- /dev/null +++ b/src/components/OnlineEvalPicker.tsx @@ -0,0 +1,93 @@ +import type { OnlineEvaluationConfigSummary } from "@aws-sdk/client-bedrock-agentcore-control"; +import { useNavigate } from "react-router"; +import type { ScreenProps } from "../handlers/types"; +import { coreOptsFromCtx } from "../handlers/utils"; +import { formatTimestamp } from "./formatTimestamp"; +import { PaginatedTablePicker } from "./PaginatedTablePicker"; +import type { DataTableColumn } from "./ui/data-table"; + +// OnlineEvalRow is the flat, display-ready shape the table renders. It also +// satisfies DataTable's `T extends Record` constraint, which the +// SDK's OnlineEvaluationConfigSummary interface does not. The list API returns +// only summary fields (name/status/executionStatus/timestamps); richer detail +// like sampling rate and evaluators comes from GetOnlineEvaluationConfig. +interface OnlineEvalRow extends Record { + configId: string; + configName: string; + status: string; + executionStatus: string; + updatedAt: string; +} + +export const onlineEvalColumns = [ + { key: "configName", header: "name", flex: true }, + { key: "status", header: "status", width: 12 }, + { key: "executionStatus", header: "execution", width: 11 }, + { + key: "updatedAt", + header: "updated UTC", + width: 16, + render: formatTimestamp, + }, +] satisfies DataTableColumn[]; + +function toRow(config: OnlineEvaluationConfigSummary): OnlineEvalRow { + const id = config.onlineEvaluationConfigId ?? ""; + return { + configId: id, + configName: config.onlineEvaluationConfigName ?? id, + status: config.status ?? "-", + executionStatus: config.executionStatus ?? "-", + updatedAt: config.updatedAt?.toISOString() ?? "-", + }; +} + +export interface OnlineEvalPickerProps extends ScreenProps { + breadcrumb: string[]; + description?: string; + onSelect: (configId: string) => void; + onEscape?: () => void; +} + +/** + * Fetches the caller's online evaluation configs and renders them as a navigable + * table. The shared body of every "pick a config" screen (list, and — in the + * write TUI — update/pause/resume/delete). Esc returns to the parent menu derived + * from the breadcrumb unless a host supplies its own onEscape. + */ +export function OnlineEvalPicker({ + ctx, + core, + breadcrumb, + description, + onSelect, + onEscape, +}: OnlineEvalPickerProps) { + const opts = coreOptsFromCtx(ctx); + const navigate = useNavigate(); + const goBack = onEscape ?? (() => navigate("/" + breadcrumb.slice(0, -1).join("/"))); + + return ( + { + const response = await core.eval.listOnlineEvaluationConfigs(token, pageSize, opts); + return { + items: response.onlineEvaluationConfigs ?? [], + nextToken: response.nextToken, + }; + }} + toRow={toRow} + columns={onlineEvalColumns} + getValue={(row) => row.configId} + onSelect={onSelect} + onBack={goBack} + loadingMessage="Loading online evaluation configs…" + errorMessage={(error) => `Error: ${error.message}`} + emptyMessage="No online evaluation configs found in this Region." + emptyPageMessage="No online evaluation configs on this page." + /> + ); +} diff --git a/src/components/Root.tsx b/src/components/Root.tsx index 0ce67fa16..ad8f804c2 100644 --- a/src/components/Root.tsx +++ b/src/components/Root.tsx @@ -32,6 +32,19 @@ import { MemoryScreen } from "../handlers/memory/screen.tsx"; import { MemoryGetJsonScreen, MemoryGetScreen } from "../handlers/memory/get/screen.tsx"; import { MemoryListScreen } from "../handlers/memory/list/screen.tsx"; import { RuntimeInvokeScreen } from "../handlers/runtime/invoke/screen.tsx"; +import { EvalScreen } from "../handlers/eval/screen.tsx"; +import { EvaluatorScreen } from "../handlers/eval/evaluator/screen.tsx"; +import { EvaluatorListScreen } from "../handlers/eval/evaluator/list/screen.tsx"; +import { + EvaluatorGetScreen, + EvaluatorGetJsonScreen, +} from "../handlers/eval/evaluator/get/screen.tsx"; +import { OnlineEvalScreen } from "../handlers/eval/online-eval/screen.tsx"; +import { OnlineEvalListScreen } from "../handlers/eval/online-eval/list/screen.tsx"; +import { + OnlineEvalGetScreen, + OnlineEvalGetJsonScreen, +} from "../handlers/eval/online-eval/get/screen.tsx"; import { MemoryEventScreen } from "../handlers/memory/event/screen.tsx"; import { MemoryEventGetScreen } from "../handlers/memory/event/get/screen.tsx"; import { MemoryEventListScreen } from "../handlers/memory/event/list/screen.tsx"; @@ -291,6 +304,48 @@ export function Root({ path, ctx, core, queryClient }: RootProps) { path="agentcore/runtime/invoke/:runtimeId/:qualifier" element={} /> + } /> + } + /> + } + /> + {/* Bare `get` (no id) has nothing to show — send the user to the list. */} + } + /> + } + /> + } + /> + } + /> + } + /> + } + /> + } + /> + } + /> } diff --git a/src/components/RouterScreen.tsx b/src/components/RouterScreen.tsx index 571b325d3..3ef34d312 100644 --- a/src/components/RouterScreen.tsx +++ b/src/components/RouterScreen.tsx @@ -44,21 +44,29 @@ export interface RouterScreenProps extends ScreenProps { // segment is the app root; the last is the command whose subcommands are the // menu options. path: string[]; + // omit hides subcommands from the menu by name. The command tree still carries + // them (they remain usable from the CLI), but they are not offered here — used + // when the TUI intentionally does not route a command yet, so it can't fall + // through to the HelpScreen catch-all (which exits the app). + omit?: string[]; } // RouterScreen renders the interactive command menu for a Router node: a filter // input at the top and the node's subcommands (read straight off the Commander // Command) as navigable options below. Selecting an option routes to that // subcommand's screen. -export function RouterScreen({ ctx, path }: RouterScreenProps) { +export function RouterScreen({ ctx, path, omit }: RouterScreenProps) { const navigate = useNavigate(); const { isRawModeSupported } = useStdin(); const { exit } = useApp(); const command = resolveCommand(ctx.require(CommandKey), path); const options: Option[] = useMemo( - () => command.commands.map((c) => ({ name: c.name(), description: c.description() })), - [command], + () => + command.commands + .filter((c) => !omit?.includes(c.name())) + .map((c) => ({ name: c.name(), description: c.description() })), + [command, omit], ); const [query, setQuery] = useState(""); diff --git a/src/handlers/eval/evaluator/evaluator.screen.test.tsx b/src/handlers/eval/evaluator/evaluator.screen.test.tsx new file mode 100644 index 000000000..9a683b448 --- /dev/null +++ b/src/handlers/eval/evaluator/evaluator.screen.test.tsx @@ -0,0 +1,220 @@ +import { afterEach, describe, expect, test } from "bun:test"; +import type { + EvaluatorSummary, + GetEvaluatorResponse, +} from "@aws-sdk/client-bedrock-agentcore-control"; +import { + cleanupScreens, + renderScreen, + TestCoreClient, + waitFor, + waitForText, +} from "../../../testing"; + +afterEach(cleanupScreens); + +const evalEndpointUrl = "https://eval.test"; + +function evaluatorSummary(overrides: Partial = {}): EvaluatorSummary { + return { + evaluatorArn: "arn:aws:bedrock-agentcore:us-east-1:123456789012:evaluator/ev-1", + evaluatorId: "ev-1", + evaluatorName: "answer_relevance", + evaluatorType: "Custom", + level: "SESSION", + status: "ACTIVE", + createdAt: new Date("2026-07-19T01:02:03.000Z"), + updatedAt: new Date("2026-07-20T12:34:56.000Z"), + lockedForModification: false, + ...overrides, + }; +} + +function getEvaluatorResponse(overrides: Partial = {}): GetEvaluatorResponse { + return { + evaluatorArn: "arn:aws:bedrock-agentcore:us-east-1:123456789012:evaluator/ev-1", + evaluatorId: "ev-1", + evaluatorName: "answer_relevance", + level: "SESSION", + status: "ACTIVE", + lockedForModification: false, + createdAt: new Date("2026-07-19T01:02:03.000Z"), + updatedAt: new Date("2026-07-20T12:34:56.000Z"), + evaluatorConfig: { + llmAsAJudge: { + instructions: "Rate whether the answer is relevant.", + ratingScale: { + numerical: [{ value: 1, label: "Poor", definition: "Fails to meet expectations" }], + }, + modelConfig: { bedrockEvaluatorModelConfig: { modelId: "anthropic.claude" } }, + }, + }, + ...overrides, + }; +} + +function coreWithEvaluators(evaluators: EvaluatorSummary[]): TestCoreClient { + const core = new TestCoreClient(); + core.eval.setListResponse({ evaluators }); + return core; +} + +describe("evaluator menu", () => { + test("offers only the read-only commands", async () => { + const screen = renderScreen("/agentcore/eval/evaluator"); + + await waitForText(screen.lastFrame, "get an evaluator by id"); + const frame = screen.lastFrame()!; + expect(frame).toContain("list"); + // Mutating subcommands are omitted so they can't fall through to HelpScreen. + expect(frame).not.toContain("llm-as-a-judge"); + expect(frame).not.toContain("code-based"); + expect(frame).not.toContain("delete"); + }); + + test("the eval root menu shows evaluator and online-eval", async () => { + const screen = renderScreen("/agentcore/eval"); + + await waitForText(screen.lastFrame, "manage AgentCore evaluators"); + expect(screen.lastFrame()).toContain("online-eval"); + }); +}); + +describe("evaluator picker", () => { + test("renders name, type, level, and update time", async () => { + const core = coreWithEvaluators([ + evaluatorSummary({ + evaluatorName: "Builtin.Correctness", + evaluatorType: "Builtin", + level: "TRACE", + updatedAt: new Date("2026-07-21T02:03:04.000Z"), + }), + ]); + const screen = renderScreen("/agentcore/eval/evaluator/list", { core }); + + await waitForText(screen.lastFrame, "Builtin.Correctness"); + const frame = screen.lastFrame()!; + expect(frame).toContain("Builtin"); + expect(frame).toContain("TRACE"); + expect(frame).toContain("2026-07-21 02:03"); + }); + + test("calls listEvaluators with exact Core options", async () => { + const core = coreWithEvaluators([evaluatorSummary()]); + renderScreen("/agentcore/eval/evaluator/list", { core, endpointUrl: evalEndpointUrl }); + + await waitFor(() => core.eval.calls.some((call) => call.method === "listEvaluators")); + expect(core.eval.calls.filter((call) => call.method === "listEvaluators")).toEqual([ + { + method: "listEvaluators", + args: [ + undefined, + expect.any(Number), + { region: "us-east-1", endpointUrl: evalEndpointUrl }, + ], + }, + ]); + }); + + test("bare evaluator get redirects to the picker", async () => { + const core = coreWithEvaluators([ + evaluatorSummary({ evaluatorId: "redirected-ev", evaluatorName: "redirected_eval" }), + ]); + const screen = renderScreen("/agentcore/eval/evaluator/get", { core }); + + await waitForText(screen.lastFrame, "redirected_eval"); + expect(core.eval.calls[0]?.method).toBe("listEvaluators"); + }); + + test("selection opens the matching evaluator detail", async () => { + const core = coreWithEvaluators([evaluatorSummary({ evaluatorId: "ev-1" })]); + core.eval.setGetResponse(getEvaluatorResponse({ evaluatorId: "ev-1" })); + const screen = renderScreen("/agentcore/eval/evaluator/list", { core }); + + await waitForText(screen.lastFrame, "answer_relevance"); + await screen.press("return"); + await waitForText(screen.lastFrame, "agentcore → eval → evaluator → get → ev-1"); + await waitFor(() => + core.eval.calls.some((call) => call.method === "getEvaluator" && call.args[0] === "ev-1"), + ); + }); + + test("shows the empty state", async () => { + const empty = renderScreen("/agentcore/eval/evaluator/list"); + await waitForText(empty.lastFrame, "No evaluators found in this Region."); + }); +}); + +describe("evaluator detail", () => { + test("renders the summary and derives the kind from the config union", async () => { + const core = new TestCoreClient(); + core.eval.setGetResponse(getEvaluatorResponse()); + const screen = renderScreen("/agentcore/eval/evaluator/get/ev-1", { + core, + endpointUrl: evalEndpointUrl, + }); + + await waitForText(screen.lastFrame, "show the full JSON definition"); + const frame = screen.lastFrame()!; + expect(frame).toContain("answer_relevance"); + expect(frame).toContain("LLM-as-a-Judge"); + expect(frame).toMatch(/level\s+SESSION/); + expect(core.eval.calls.find((call) => call.method === "getEvaluator")).toEqual({ + method: "getEvaluator", + args: ["ev-1", { region: "us-east-1", endpointUrl: evalEndpointUrl }], + }); + }); + + test("labels a code-based evaluator", async () => { + const core = new TestCoreClient(); + core.eval.setGetResponse( + getEvaluatorResponse({ + evaluatorConfig: { + codeBased: { lambdaConfig: { lambdaArn: "arn:aws:lambda:us-east-1:1234:function:x" } }, + }, + }), + ); + const screen = renderScreen("/agentcore/eval/evaluator/get/ev-1", { core }); + + await waitForText(screen.lastFrame, "code-based"); + }); + + test("shows a locked marker only when locked", async () => { + const unlocked = new TestCoreClient(); + unlocked.eval.setGetResponse(getEvaluatorResponse({ lockedForModification: false })); + const open = renderScreen("/agentcore/eval/evaluator/get/ev-1", { core: unlocked }); + await waitForText(open.lastFrame, "show the full JSON definition"); + expect(open.lastFrame()).not.toContain("locked"); + open.unmount(); + + const lockedCore = new TestCoreClient(); + lockedCore.eval.setGetResponse(getEvaluatorResponse({ lockedForModification: true })); + const locked = renderScreen("/agentcore/eval/evaluator/get/ev-1", { core: lockedCore }); + await waitForText(locked.lastFrame, "locked"); + }); + + test("opens the complete evaluator JSON", async () => { + const core = new TestCoreClient(); + core.eval.setGetResponse(getEvaluatorResponse()); + const screen = renderScreen("/agentcore/eval/evaluator/get/ev-1", { core }); + + await waitForText(screen.lastFrame, "show the full JSON definition"); + await screen.press("return"); + await waitForText(screen.lastFrame, "agentcore → eval → evaluator → get → ev-1 → json"); + expect(screen.lastFrame()).toContain('"instructions"'); + }); + + test("retries a failed detail query", async () => { + const core = new TestCoreClient(); + core.eval.setError(new Error("evaluator unavailable")); + const screen = renderScreen("/agentcore/eval/evaluator/get/ev-1", { core }); + + await waitForText(screen.lastFrame, "evaluator unavailable"); + expect(screen.lastFrame()).toContain("[r] retry"); + + core.eval.setError(undefined); + core.eval.setGetResponse(getEvaluatorResponse()); + await screen.write("r"); + await waitForText(screen.lastFrame, "show the full JSON definition"); + }); +}); diff --git a/src/handlers/eval/evaluator/evaluator.test.tsx b/src/handlers/eval/evaluator/evaluator.test.tsx index 4d691d16c..bb41c3f4f 100644 --- a/src/handlers/eval/evaluator/evaluator.test.tsx +++ b/src/handlers/eval/evaluator/evaluator.test.tsx @@ -141,16 +141,30 @@ describe("eval command hierarchy", () => { ).toEqual(["create", "update"]); }); + // Under --json the empty-invocation TUI middleware prints help instead of + // opening the interactive UI, so these exercise the headless path. test.each([ "eval", "eval evaluator", "eval evaluator llm-as-a-judge", "eval evaluator code-based", - ])("prints help for bare `%s` without an SDK call", async (command) => { - const stdout = await run(command.split(" ")); + ])("prints help for bare `%s --json` without an SDK call", async (command) => { + const stdout = await run([...command.split(" "), "--json"]); expect(stdout).toContain(`Usage: agentcore ${command}`); expect(stdout).toContain("Commands:"); }); + + // A bare read leaf (no flags, no --json) opens the interactive TUI, which the + // headless test IO cannot host — proving the empty-invocation middleware is + // wired onto the evaluator commands. + test.each([["get"], ["list"]] as const)( + "opens the TUI for a bare `eval evaluator %s` leaf", + async (command) => { + await expect(run(["eval", "evaluator", command])).rejects.toThrow( + "interactive mode requires a TTY on stdin and stdout", + ); + }, + ); }); describe("evaluator CRUDL", () => { @@ -199,7 +213,9 @@ describe("evaluator CRUDL", () => { }); test("lists evaluators", async () => { - const stdout = await run(["eval", "evaluator", "list"]); + // --json forces the headless path; a bare `list` (no flags) would otherwise + // open the TUI under the empty-invocation middleware. + const stdout = await run(["eval", "evaluator", "list", "--json"]); matchGolden(FIXTURES, "list.golden.json", stdout); expect(JSON.parse(stdout).evaluators).toBeArray(); @@ -377,13 +393,15 @@ describe("evaluator flag validation", () => { ).rejects.toThrow(/--lambda-arn/); }); + // --json forces the headless path so the required-flag error surfaces; without + // it a bare invocation opens the TUI under the empty-invocation middleware. test.each([ ["llm-as-a-judge update", ["eval", "evaluator", "llm-as-a-judge", "update"]], ["code-based update", ["eval", "evaluator", "code-based", "update"]], ["get", ["eval", "evaluator", "get"]], ["delete", ["eval", "evaluator", "delete"]], ] as const)("`%s` requires --id", async (_label, args) => { - await expect(run([...args])).rejects.toThrow(/--id/); + await expect(run([...args, "--json"])).rejects.toThrow(/--id/); }); test("rejects malformed custom rating scale JSON", async () => { diff --git a/src/handlers/eval/evaluator/get/screen.tsx b/src/handlers/eval/evaluator/get/screen.tsx new file mode 100644 index 000000000..038149fdb --- /dev/null +++ b/src/handlers/eval/evaluator/get/screen.tsx @@ -0,0 +1,80 @@ +import { useQuery } from "@tanstack/react-query"; +import { useNavigate, useParams } from "react-router"; +import type { GetEvaluatorResponse } from "@aws-sdk/client-bedrock-agentcore-control"; +import { JsonDetail } from "../../../../components/JsonDetail"; +import { ResourceDetailScreen } from "../../../../components/ResourceDetailScreen"; +import type { ScreenProps } from "../../../types"; +import { coreOptsFromCtx } from "../../../utils"; + +function useEvaluatorDetail({ ctx, core }: ScreenProps, evaluatorId: string | undefined) { + const opts = coreOptsFromCtx(ctx); + return useQuery({ + queryKey: ["evaluator", opts.region, evaluatorId], + queryFn: () => core.eval.getEvaluator(evaluatorId!, opts), + enabled: evaluatorId !== undefined, + }); +} + +// evaluatorKind names the arm of the evaluatorConfig union in display terms. The +// GetEvaluator response has no type field of its own (unlike the list summary); +// the kind is which arm of the config is populated. +function evaluatorKind(evaluator: GetEvaluatorResponse | undefined): string { + if (evaluator?.evaluatorConfig?.llmAsAJudge) return "LLM-as-a-Judge"; + if (evaluator?.evaluatorConfig?.codeBased) return "code-based"; + return "-"; +} + +export function EvaluatorGetScreen(props: ScreenProps) { + const navigate = useNavigate(); + const { evaluatorId } = useParams(); + const detail = useEvaluatorDetail(props, evaluatorId); + const evaluator = detail.data; + + return ( + + navigate(`/agentcore/eval/evaluator/get/${encodeURIComponent(evaluatorId)}/json`), + }, + ] + : [] + } + loadingLabel="Loading evaluator…" + onRetry={() => void detail.refetch()} + selectLabel="open detail" + /> + ); +} + +export function EvaluatorGetJsonScreen(props: ScreenProps) { + const { evaluatorId } = useParams(); + const detail = useEvaluatorDetail(props, evaluatorId); + + return ( + void detail.refetch()} + /> + ); +} diff --git a/src/handlers/eval/evaluator/index.tsx b/src/handlers/eval/evaluator/index.tsx index 548c1c03e..420e2ebed 100644 --- a/src/handlers/eval/evaluator/index.tsx +++ b/src/handlers/eval/evaluator/index.tsx @@ -1,7 +1,8 @@ import { Router } from "../../../router"; +import { renderTui } from "../../../tui"; +import { withTuiOnEmptyFlagsAndArgs } from "../../../middleware"; import type { AppIO } from "../../../io"; import type { Core } from "../../types"; -import { createHelpDefault } from "../../help"; import { createLlmAsAJudgeHandler } from "./llm-as-a-judge"; import { createCodeBasedHandler } from "./code-based"; import { createGetEvaluatorHandler } from "./get"; @@ -10,10 +11,13 @@ import { createDeleteEvaluatorHandler } from "./delete"; export function createEvaluatorHandler(core: Core, io: AppIO): Router { return new Router("evaluator", "manage AgentCore evaluators") - .default(createHelpDefault(io)) + .use(withTuiOnEmptyFlagsAndArgs(core, io)) + .default(renderTui(core, io)) .handler(createLlmAsAJudgeHandler(core, io)) .handler(createCodeBasedHandler(core, io)) .handler(createGetEvaluatorHandler(core)) .handler(createListEvaluatorsHandler(core)) .handler(createDeleteEvaluatorHandler(core)); } + +export { EvaluatorScreen } from "./screen.tsx"; diff --git a/src/handlers/eval/evaluator/list/screen.tsx b/src/handlers/eval/evaluator/list/screen.tsx new file mode 100644 index 000000000..214fe69a3 --- /dev/null +++ b/src/handlers/eval/evaluator/list/screen.tsx @@ -0,0 +1,17 @@ +import { useNavigate } from "react-router"; +import { EvaluatorPicker } from "../../../../components/EvaluatorPicker"; +import type { ScreenProps } from "../../../types"; + +export function EvaluatorListScreen(props: ScreenProps) { + const navigate = useNavigate(); + + return ( + + navigate(`/agentcore/eval/evaluator/get/${encodeURIComponent(evaluatorId)}`) + } + /> + ); +} diff --git a/src/handlers/eval/evaluator/screen.tsx b/src/handlers/eval/evaluator/screen.tsx new file mode 100644 index 000000000..07fc43f6b --- /dev/null +++ b/src/handlers/eval/evaluator/screen.tsx @@ -0,0 +1,12 @@ +import { RouterScreen } from "../../../components/RouterScreen"; +import type { ScreenProps } from "../../types"; + +// The read-only TUI offers only get/list. The mutating subcommands +// (llm-as-a-judge and code-based, which host create/update; and delete) stay +// CLI-only for now, so they are omitted from the menu — an unrouted menu entry +// would fall through to the HelpScreen catch-all and exit the app. +const OMIT = ["llm-as-a-judge", "code-based", "delete"]; + +export function EvaluatorScreen(props: ScreenProps) { + return ; +} diff --git a/src/handlers/eval/index.tsx b/src/handlers/eval/index.tsx index 1b14b201d..d606425a5 100644 --- a/src/handlers/eval/index.tsx +++ b/src/handlers/eval/index.tsx @@ -1,13 +1,17 @@ import { Router } from "../../router"; +import { renderTui } from "../../tui"; +import { withTuiOnEmptyFlagsAndArgs } from "../../middleware"; import type { AppIO } from "../../io"; import type { Core } from "../types"; -import { createHelpDefault } from "../help"; import { createEvaluatorHandler } from "./evaluator"; import { createOnlineEvalHandler } from "./online-eval"; export function createEvalHandler(core: Core, io: AppIO): Router { return new Router("eval", "evaluate and optimize AgentCore agents") - .default(createHelpDefault(io)) + .use(withTuiOnEmptyFlagsAndArgs(core, io)) + .default(renderTui(core, io)) .handler(createEvaluatorHandler(core, io)) .handler(createOnlineEvalHandler(core, io)); } + +export { EvalScreen } from "./screen.tsx"; diff --git a/src/handlers/eval/online-eval/get/screen.tsx b/src/handlers/eval/online-eval/get/screen.tsx new file mode 100644 index 000000000..44063a597 --- /dev/null +++ b/src/handlers/eval/online-eval/get/screen.tsx @@ -0,0 +1,72 @@ +import { useQuery } from "@tanstack/react-query"; +import { useNavigate, useParams } from "react-router"; +import { JsonDetail } from "../../../../components/JsonDetail"; +import { ResourceDetailScreen } from "../../../../components/ResourceDetailScreen"; +import type { ScreenProps } from "../../../types"; +import { coreOptsFromCtx } from "../../../utils"; + +function useOnlineEvalDetail({ ctx, core }: ScreenProps, configId: string | undefined) { + const opts = coreOptsFromCtx(ctx); + return useQuery({ + queryKey: ["online-eval", opts.region, configId], + queryFn: () => core.eval.getOnlineEvaluationConfig(configId!, opts), + enabled: configId !== undefined, + }); +} + +export function OnlineEvalGetScreen(props: ScreenProps) { + const navigate = useNavigate(); + const { configId } = useParams(); + const detail = useOnlineEvalDetail(props, configId); + const config = detail.data; + const samplingPercentage = config?.rule?.samplingConfig?.samplingPercentage; + + return ( + + navigate(`/agentcore/eval/online-eval/get/${encodeURIComponent(configId)}/json`), + }, + ] + : [] + } + loadingLabel="Loading online evaluation config…" + onRetry={() => void detail.refetch()} + selectLabel="open detail" + /> + ); +} + +export function OnlineEvalGetJsonScreen(props: ScreenProps) { + const { configId } = useParams(); + const detail = useOnlineEvalDetail(props, configId); + + return ( + void detail.refetch()} + /> + ); +} diff --git a/src/handlers/eval/online-eval/index.tsx b/src/handlers/eval/online-eval/index.tsx index b37d07662..a221f70f2 100644 --- a/src/handlers/eval/online-eval/index.tsx +++ b/src/handlers/eval/online-eval/index.tsx @@ -1,7 +1,8 @@ import { Router } from "../../../router"; +import { renderTui } from "../../../tui"; +import { withTuiOnEmptyFlagsAndArgs } from "../../../middleware"; import type { AppIO } from "../../../io"; import type { Core } from "../../types"; -import { createHelpDefault } from "../../help"; import { createCreateOnlineEvalHandler } from "./create"; import { createGetOnlineEvalHandler } from "./get"; import { createListOnlineEvalHandler } from "./list"; @@ -12,7 +13,8 @@ import { createDeleteOnlineEvalHandler } from "./delete"; export function createOnlineEvalHandler(core: Core, io: AppIO): Router { return new Router("online-eval", "manage AgentCore online evaluation configs") - .default(createHelpDefault(io)) + .use(withTuiOnEmptyFlagsAndArgs(core, io)) + .default(renderTui(core, io)) .handler(createCreateOnlineEvalHandler(core, io)) .handler(createGetOnlineEvalHandler(core)) .handler(createListOnlineEvalHandler(core)) @@ -21,3 +23,5 @@ export function createOnlineEvalHandler(core: Core, io: AppIO): Router { .handler(createResumeOnlineEvalHandler(core)) .handler(createDeleteOnlineEvalHandler(core)); } + +export { OnlineEvalScreen } from "./screen.tsx"; diff --git a/src/handlers/eval/online-eval/list/screen.tsx b/src/handlers/eval/online-eval/list/screen.tsx new file mode 100644 index 000000000..be707838d --- /dev/null +++ b/src/handlers/eval/online-eval/list/screen.tsx @@ -0,0 +1,17 @@ +import { useNavigate } from "react-router"; +import { OnlineEvalPicker } from "../../../../components/OnlineEvalPicker"; +import type { ScreenProps } from "../../../types"; + +export function OnlineEvalListScreen(props: ScreenProps) { + const navigate = useNavigate(); + + return ( + + navigate(`/agentcore/eval/online-eval/get/${encodeURIComponent(configId)}`) + } + /> + ); +} diff --git a/src/handlers/eval/online-eval/online-eval.screen.test.tsx b/src/handlers/eval/online-eval/online-eval.screen.test.tsx new file mode 100644 index 000000000..82e25a12c --- /dev/null +++ b/src/handlers/eval/online-eval/online-eval.screen.test.tsx @@ -0,0 +1,194 @@ +import { afterEach, describe, expect, test } from "bun:test"; +import type { + GetOnlineEvaluationConfigResponse, + OnlineEvaluationConfigSummary, +} from "@aws-sdk/client-bedrock-agentcore-control"; +import { + cleanupScreens, + renderScreen, + TestCoreClient, + waitFor, + waitForText, +} from "../../../testing"; + +afterEach(cleanupScreens); + +const evalEndpointUrl = "https://eval.test"; + +function configSummary( + overrides: Partial = {}, +): OnlineEvaluationConfigSummary { + return { + onlineEvaluationConfigArn: + "arn:aws:bedrock-agentcore:us-east-1:123456789012:online-evaluation-config/oec-1", + onlineEvaluationConfigId: "oec-1", + onlineEvaluationConfigName: "prod_quality_watch", + status: "ACTIVE", + executionStatus: "ENABLED", + createdAt: new Date("2026-07-19T01:02:03.000Z"), + updatedAt: new Date("2026-07-20T12:34:56.000Z"), + ...overrides, + }; +} + +function getConfigResponse( + overrides: Partial = {}, +): GetOnlineEvaluationConfigResponse { + return { + onlineEvaluationConfigArn: + "arn:aws:bedrock-agentcore:us-east-1:123456789012:online-evaluation-config/oec-1", + onlineEvaluationConfigId: "oec-1", + onlineEvaluationConfigName: "prod_quality_watch", + status: "ACTIVE", + executionStatus: "ENABLED", + rule: { samplingConfig: { samplingPercentage: 5 } }, + dataSourceConfig: { + cloudWatchLogs: { logGroupNames: ["/aws/bedrock-agentcore/runtime/x"], serviceNames: [] }, + }, + evaluators: [{ evaluatorId: "ev-1" }], + evaluationExecutionRoleArn: "arn:aws:iam::123456789012:role/online-eval-role", + createdAt: new Date("2026-07-19T01:02:03.000Z"), + updatedAt: new Date("2026-07-20T12:34:56.000Z"), + ...overrides, + }; +} + +function coreWithConfigs(configs: OnlineEvaluationConfigSummary[]): TestCoreClient { + const core = new TestCoreClient(); + core.eval.setOnlineEvalListResponse({ onlineEvaluationConfigs: configs }); + return core; +} + +describe("online-eval menu", () => { + test("offers only the read-only commands", async () => { + const screen = renderScreen("/agentcore/eval/online-eval"); + + await waitForText(screen.lastFrame, "get an online evaluation config by id"); + const frame = screen.lastFrame()!; + expect(frame).toContain("list"); + expect(frame).not.toContain("create"); + expect(frame).not.toContain("update"); + expect(frame).not.toContain("pause"); + expect(frame).not.toContain("resume"); + expect(frame).not.toContain("delete"); + }); +}); + +describe("online-eval picker", () => { + test("renders name, status, execution status, and update time", async () => { + const core = coreWithConfigs([ + configSummary({ + onlineEvaluationConfigName: "staging_regression", + executionStatus: "DISABLED", + updatedAt: new Date("2026-07-21T02:03:04.000Z"), + }), + ]); + const screen = renderScreen("/agentcore/eval/online-eval/list", { core }); + + await waitForText(screen.lastFrame, "staging_regression"); + const frame = screen.lastFrame()!; + expect(frame).toContain("DISABLED"); + expect(frame).toContain("2026-07-21 02:03"); + }); + + test("calls listOnlineEvaluationConfigs with exact Core options", async () => { + const core = coreWithConfigs([configSummary()]); + renderScreen("/agentcore/eval/online-eval/list", { core, endpointUrl: evalEndpointUrl }); + + await waitFor(() => + core.eval.calls.some((call) => call.method === "listOnlineEvaluationConfigs"), + ); + expect(core.eval.calls.filter((call) => call.method === "listOnlineEvaluationConfigs")).toEqual( + [ + { + method: "listOnlineEvaluationConfigs", + args: [ + undefined, + expect.any(Number), + { region: "us-east-1", endpointUrl: evalEndpointUrl }, + ], + }, + ], + ); + }); + + test("bare online-eval get redirects to the picker", async () => { + const core = coreWithConfigs([ + configSummary({ + onlineEvaluationConfigId: "redirected-oec", + onlineEvaluationConfigName: "redirected_config", + }), + ]); + const screen = renderScreen("/agentcore/eval/online-eval/get", { core }); + + await waitForText(screen.lastFrame, "redirected_config"); + expect(core.eval.calls[0]?.method).toBe("listOnlineEvaluationConfigs"); + }); + + test("selection opens the matching config detail", async () => { + const core = coreWithConfigs([configSummary({ onlineEvaluationConfigId: "oec-1" })]); + core.eval.setOnlineEvalGetResponse(getConfigResponse({ onlineEvaluationConfigId: "oec-1" })); + const screen = renderScreen("/agentcore/eval/online-eval/list", { core }); + + await waitForText(screen.lastFrame, "prod_quality_watch"); + await screen.press("return"); + await waitForText(screen.lastFrame, "agentcore → eval → online-eval → get → oec-1"); + await waitFor(() => + core.eval.calls.some( + (call) => call.method === "getOnlineEvaluationConfig" && call.args[0] === "oec-1", + ), + ); + }); + + test("shows the empty state", async () => { + const empty = renderScreen("/agentcore/eval/online-eval/list"); + await waitForText(empty.lastFrame, "No online evaluation configs found in this Region."); + }); +}); + +describe("online-eval detail", () => { + test("renders sampling, execution status, and evaluator count", async () => { + const core = new TestCoreClient(); + core.eval.setOnlineEvalGetResponse(getConfigResponse()); + const screen = renderScreen("/agentcore/eval/online-eval/get/oec-1", { + core, + endpointUrl: evalEndpointUrl, + }); + + await waitForText(screen.lastFrame, "show the full JSON"); + const frame = screen.lastFrame()!; + expect(frame).toContain("prod_quality_watch"); + expect(frame).toMatch(/sampling\s+5%/); + expect(frame).toMatch(/execution\s+ENABLED/); + expect(frame).toMatch(/evaluators\s+1/); + expect(core.eval.calls.find((call) => call.method === "getOnlineEvaluationConfig")).toEqual({ + method: "getOnlineEvaluationConfig", + args: ["oec-1", { region: "us-east-1", endpointUrl: evalEndpointUrl }], + }); + }); + + test("opens the complete config JSON", async () => { + const core = new TestCoreClient(); + core.eval.setOnlineEvalGetResponse(getConfigResponse()); + const screen = renderScreen("/agentcore/eval/online-eval/get/oec-1", { core }); + + await waitForText(screen.lastFrame, "show the full JSON"); + await screen.press("return"); + await waitForText(screen.lastFrame, "agentcore → eval → online-eval → get → oec-1 → json"); + expect(screen.lastFrame()).toContain('"samplingConfig"'); + }); + + test("retries a failed detail query", async () => { + const core = new TestCoreClient(); + core.eval.setError(new Error("config unavailable")); + const screen = renderScreen("/agentcore/eval/online-eval/get/oec-1", { core }); + + await waitForText(screen.lastFrame, "config unavailable"); + expect(screen.lastFrame()).toContain("[r] retry"); + + core.eval.setError(undefined); + core.eval.setOnlineEvalGetResponse(getConfigResponse()); + await screen.write("r"); + await waitForText(screen.lastFrame, "show the full JSON"); + }); +}); diff --git a/src/handlers/eval/online-eval/online-eval.test.tsx b/src/handlers/eval/online-eval/online-eval.test.tsx index 8b9659d0a..1e703d61d 100644 --- a/src/handlers/eval/online-eval/online-eval.test.tsx +++ b/src/handlers/eval/online-eval/online-eval.test.tsx @@ -92,11 +92,25 @@ describe("eval online-eval command hierarchy", () => { ]); }); - test("prints help for bare `eval online-eval` without an SDK call", async () => { - const stdout = await run(["eval", "online-eval"]); + test("prints help for bare `eval online-eval --json` without an SDK call", async () => { + // Under --json the empty-invocation TUI middleware prints help instead of + // opening the interactive UI. + const stdout = await run(["eval", "online-eval", "--json"]); expect(stdout).toContain("Usage: agentcore eval online-eval"); expect(stdout).toContain("Commands:"); }); + + // A bare read leaf (no flags, no --json) opens the interactive TUI, which the + // headless test IO cannot host — proving the empty-invocation middleware is + // wired onto the online-eval commands. + test.each([["get"], ["list"]] as const)( + "opens the TUI for a bare `eval online-eval %s` leaf", + async (command) => { + await expect(run(["eval", "online-eval", command])).rejects.toThrow( + "interactive mode requires a TTY on stdin and stdout", + ); + }, + ); }); describe("online-eval CRUDL", () => { @@ -125,7 +139,9 @@ describe("online-eval CRUDL", () => { }); test("lists online evaluation configs", async () => { - const stdout = await run(["eval", "online-eval", "list"]); + // --json forces the headless path; a bare `list` (no flags) would otherwise + // open the TUI under the empty-invocation middleware. + const stdout = await run(["eval", "online-eval", "list", "--json"]); matchGolden(FIXTURES, "list.golden.json", stdout); expect(JSON.parse(stdout).onlineEvaluationConfigs).toBeArray(); @@ -350,8 +366,10 @@ describe("flag validation", () => { ).rejects.toThrow(/'--endpoint' cannot be combined with '--data-source-config'/); }); + // --json forces the headless path so the required-flag error surfaces; without + // it a bare invocation opens the TUI under the empty-invocation middleware. test.each(["get", "update", "pause", "resume", "delete"])("%s requires --id", async (command) => { - await expect(run(["eval", "online-eval", command])).rejects.toThrow( + await expect(run(["eval", "online-eval", command, "--json"])).rejects.toThrow( /required option '--id ' not specified/, ); }); diff --git a/src/handlers/eval/online-eval/screen.tsx b/src/handlers/eval/online-eval/screen.tsx new file mode 100644 index 000000000..2de202490 --- /dev/null +++ b/src/handlers/eval/online-eval/screen.tsx @@ -0,0 +1,11 @@ +import { RouterScreen } from "../../../components/RouterScreen"; +import type { ScreenProps } from "../../types"; + +// The read-only TUI offers only get/list. The mutating subcommands stay CLI-only +// for now, so they are omitted from the menu — an unrouted menu entry would fall +// through to the HelpScreen catch-all and exit the app. +const OMIT = ["create", "update", "pause", "resume", "delete"]; + +export function OnlineEvalScreen(props: ScreenProps) { + return ; +} diff --git a/src/handlers/eval/screen.tsx b/src/handlers/eval/screen.tsx new file mode 100644 index 000000000..6f3af472d --- /dev/null +++ b/src/handlers/eval/screen.tsx @@ -0,0 +1,6 @@ +import { RouterScreen } from "../../components/RouterScreen"; +import type { ScreenProps } from "../types"; + +export function EvalScreen(props: ScreenProps) { + return ; +}