From 997b6b2bd9aee441337249d2cdcdc2f2e4b6a915 Mon Sep 17 00:00:00 2001 From: Tom Alexander Date: Wed, 12 Aug 2026 12:57:18 -0400 Subject: [PATCH 1/8] fix(common-utils): map NULL-literal SELECT aliases `chSqlToAliasMap` walks the parsed SELECT and records each alias against the expression behind it. A NULL literal (`NULL AS "x"`) parses without a source location, so it fell through every branch and was dropped from the map. Callers then treated the alias as a real column: the row-identity WHERE builder, for instance, emitted `x = ...` against a column that does not exist rather than `isNull(NULL)`. --- .changeset/null-literal-select-alias.md | 9 +++++++++ .../src/__tests__/clickhouse.test.ts | 16 ++++++++++++++++ packages/common-utils/src/clickhouse/index.ts | 4 ++++ 3 files changed, 29 insertions(+) create mode 100644 .changeset/null-literal-select-alias.md diff --git a/.changeset/null-literal-select-alias.md b/.changeset/null-literal-select-alias.md new file mode 100644 index 0000000000..843c125b0d --- /dev/null +++ b/.changeset/null-literal-select-alias.md @@ -0,0 +1,9 @@ +--- +'@hyperdx/common-utils': patch +--- + +Fix `NULL AS "alias"` projections being dropped from the SELECT alias map. A +column projected as a NULL literal was omitted, so anything resolving a value +back to its source expression (for example building a WHERE clause that +identifies a specific row) treated the alias as a real table column and +produced SQL referencing a column that does not exist. diff --git a/packages/common-utils/src/__tests__/clickhouse.test.ts b/packages/common-utils/src/__tests__/clickhouse.test.ts index dcec597f09..f8e2314dc4 100644 --- a/packages/common-utils/src/__tests__/clickhouse.test.ts +++ b/packages/common-utils/src/__tests__/clickhouse.test.ts @@ -185,6 +185,22 @@ describe('chSqlToAliasMap - alias unit test', () => { expect(res).toEqual(aliasMap); }); + it('NULL literal alias (multi-source padding column)', () => { + const chSqlInput: ChSql = { + sql: 'SELECT Timestamp as "__hdx_timestamp", NULL as "__hdx_duration_ms" FROM {HYPERDX_PARAM_1544803905:Identifier}.{HYPERDX_PARAM_129845054:Identifier} ORDER BY Timestamp DESC LIMIT {HYPERDX_PARAM_49586:Int32}', + params: { + HYPERDX_PARAM_1544803905: 'default', + HYPERDX_PARAM_129845054: 'otel_logs', + HYPERDX_PARAM_49586: 200, + }, + }; + const res = chSqlToAliasMap(chSqlInput); + expect(res).toEqual({ + __hdx_timestamp: 'Timestamp', + __hdx_duration_ms: 'NULL', + }); + }); + it('Normal alias, with brackets', () => { const chSqlInput: ChSql = { sql: "SELECT Timestamp as ts,ResourceAttributes['service.name'] as serviceTest,Body,TimestampTime,ServiceName,TimestampTime FROM {HYPERDX_PARAM_1544803905:Identifier}.{HYPERDX_PARAM_129845054:Identifier} WHERE (TimestampTime >= fromUnixTimestamp64Milli({HYPERDX_PARAM_1456399765:Int64}) AND TimestampTime <= fromUnixTimestamp64Milli({HYPERDX_PARAM_1719057412:Int64})) ORDER BY TimestampTime DESC LIMIT {HYPERDX_PARAM_49586:Int32} OFFSET {HYPERDX_PARAM_48:Int32}", diff --git a/packages/common-utils/src/clickhouse/index.ts b/packages/common-utils/src/clickhouse/index.ts index 4bdd8ba097..1583846e2a 100644 --- a/packages/common-utils/src/clickhouse/index.ts +++ b/packages/common-utils/src/clickhouse/index.ts @@ -1003,6 +1003,10 @@ function selectColumnsToAliasMap( `${column.expr.column.expr.value}['${column.expr.array_index[0].index.value}']` : // normal alias column.expr.column.expr.value; + } else if (column.expr.type === 'null') { + // NULL literal projection (multi-source search pads columns a source + // lacks with `NULL AS "alias"`); the parser emits it without a loc. + aliasMap[column.as] = 'NULL'; } else if (column.expr.loc != null) { aliasMap[column.as] = parsedSql.slice( column.expr.loc.start.offset, From e00bda7d5d8c5618c5acaf0fdc5726b208b0613b Mon Sep 17 00:00:00 2001 From: Tom Alexander Date: Wed, 12 Aug 2026 12:58:14 -0400 Subject: [PATCH 2/8] feat(common-utils): canonical SELECT builder for cross-source search MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Searching several sources at once needs their rows to share a shape, but each source names its own columns. This builds a SELECT per source that projects that source's semantic expressions under shared aliases — timestamp, service, severity (status for traces), body (span name for traces), duration — and pads anything a source lacks with NULL so every source returns the same columns. Callers can also request extra columns; each resolves to the column where the source has it and NULL where it doesn't. Nothing calls this yet. --- .changeset/multi-source-select-builder.md | 9 ++ .../core/__tests__/searchChartConfig.test.ts | 120 +++++++++++++++ .../src/core/searchChartConfig.ts | 141 ++++++++++++++++++ 3 files changed, 270 insertions(+) create mode 100644 .changeset/multi-source-select-builder.md diff --git a/.changeset/multi-source-select-builder.md b/.changeset/multi-source-select-builder.md new file mode 100644 index 0000000000..acfa7119e0 --- /dev/null +++ b/.changeset/multi-source-select-builder.md @@ -0,0 +1,9 @@ +--- +'@hyperdx/common-utils': minor +--- + +Add a builder for search queries that project a canonical, source-independent +column set. Given a source, it emits that source's semantic expressions +(timestamp, service, severity/status, body/span name, duration) under shared +aliases, and pads columns a source doesn't have with NULL, so results from +different tables share one shape. diff --git a/packages/common-utils/src/core/__tests__/searchChartConfig.test.ts b/packages/common-utils/src/core/__tests__/searchChartConfig.test.ts index 99d28a5c53..aff89d08ce 100644 --- a/packages/common-utils/src/core/__tests__/searchChartConfig.test.ts +++ b/packages/common-utils/src/core/__tests__/searchChartConfig.test.ts @@ -1,6 +1,9 @@ import { ALERT_COUNT_DEFAULT_SELECT, + buildMultiSourceSearchConfig, + buildMultiSourceSelect, buildSearchChartConfig, + MULTI_SOURCE_ALIASES, } from '@/core/searchChartConfig'; import { DisplayType, Filter, SourceKind, TSource } from '@/types'; @@ -477,3 +480,120 @@ describe('buildSearchChartConfig', () => { }); }); }); + +describe('buildMultiSourceSelect', () => { + it('projects every canonical alias, so callers can rely on the constants', () => { + const select = buildMultiSourceSelect(makeTraceSource(), { + includeDuration: true, + }); + + for (const alias of Object.values(MULTI_SOURCE_ALIASES)) { + expect(select).toContain(`AS "${alias}"`); + } + }); + + it('projects the canonical aliases from a Log source semantic expressions', () => { + const source = makeLogSource({ + displayedTimestampValueExpression: 'Timestamp', + serviceNameExpression: 'ServiceName', + severityTextExpression: 'SeverityText', + bodyExpression: 'Body', + }); + + expect(buildMultiSourceSelect(source)).toBe( + 'Timestamp AS "__hdx_timestamp", ' + + 'ServiceName AS "__hdx_service", ' + + 'SeverityText AS "__hdx_severity", ' + + 'Body AS "__hdx_body"', + ); + }); + + it('falls back to the first timestamp expression and NULL for missing semantics', () => { + const source = makeLogSource({ + timestampValueExpression: 'TimestampTime, Timestamp', + implicitColumnExpression: undefined, + }); + + expect(buildMultiSourceSelect(source)).toBe( + 'TimestampTime AS "__hdx_timestamp", ' + + 'NULL AS "__hdx_service", ' + + 'NULL AS "__hdx_severity", ' + + 'NULL AS "__hdx_body"', + ); + }); + + it('maps Trace sources onto status/span-name and a milliseconds duration', () => { + const source = makeTraceSource({ + serviceNameExpression: 'ServiceName', + statusCodeExpression: 'StatusCode', + spanNameExpression: 'SpanName', + durationExpression: 'Duration', + durationPrecision: 9, + }); + + expect(buildMultiSourceSelect(source, { includeDuration: true })).toBe( + 'Timestamp AS "__hdx_timestamp", ' + + 'ServiceName AS "__hdx_service", ' + + 'StatusCode AS "__hdx_severity", ' + + 'SpanName AS "__hdx_body", ' + + '(Duration)/1e6 AS "__hdx_duration_ms"', + ); + }); + + it('projects NULL duration for Log sources when duration is included', () => { + const source = makeLogSource({ bodyExpression: 'Body' }); + + expect(buildMultiSourceSelect(source, { includeDuration: true })).toContain( + 'NULL AS "__hdx_duration_ms"', + ); + }); + + it('appends extra columns, projecting NULL where a source lacks the column', () => { + const source = makeLogSource({ bodyExpression: 'Body' }); + + const select = buildMultiSourceSelect(source, { + extraColumns: [ + { name: 'ServiceName', expression: 'ServiceName' }, + { name: 'StatusCode', expression: null }, + ], + }); + + expect(select).toContain('ServiceName AS "ServiceName"'); + expect(select).toContain('NULL AS "StatusCode"'); + }); +}); + +describe('buildMultiSourceSearchConfig', () => { + it('keeps the standard search config assembly but swaps in the canonical SELECT', () => { + const source = makeLogSource({ + bodyExpression: 'Body', + tableFilterExpression: "ServiceName != 'noisy'", + }); + + const config = buildMultiSourceSearchConfig(source, { + where: 'error', + whereLanguage: 'lucene', + orderBy: 'TimestampTime DESC', + }); + + expect(config.select).toBe(buildMultiSourceSelect(source)); + expect(config.from).toEqual(source.from); + expect(config.connection).toBe('conn-1'); + expect(config.where).toBe('error'); + expect(config.whereLanguage).toBe('lucene'); + expect(config.orderBy).toBe('TimestampTime DESC'); + // Source-level behaviors (e.g. tableFilterExpression) still apply. + expect(config.filters).toEqual([ + { type: 'sql', condition: "ServiceName != 'noisy'" }, + ]); + }); + + it('never resolves to defaultTableSelectExpression', () => { + const config = buildMultiSourceSearchConfig(makeTraceSource(), { + where: '', + }); + + expect(config.select).not.toContain('SpanName,'); + expect(config.select).toContain('AS "__hdx_timestamp"'); + }); +}); diff --git a/packages/common-utils/src/core/searchChartConfig.ts b/packages/common-utils/src/core/searchChartConfig.ts index 37144db8ab..04d7dcfdca 100644 --- a/packages/common-utils/src/core/searchChartConfig.ts +++ b/packages/common-utils/src/core/searchChartConfig.ts @@ -1,3 +1,4 @@ +import { getFirstTimestampValueExpression } from '@/core/utils'; import { BuilderChartConfig, DateRange, @@ -185,3 +186,143 @@ export function buildSearchChartConfig( return config; } + +/** + * Canonical result-column aliases used when searching across multiple sources + * at once. Every selected source's SELECT is rewritten to this shape, so the + * merged results table can map columns by name regardless of how each source's + * underlying schema names them. + * + * The names are quoted aliases (`expr AS "__hdx_timestamp"`), so ClickHouse + * returns them verbatim — unlike raw expressions, which CH may reformat. + */ +export const MULTI_SOURCE_ALIASES = { + timestamp: '__hdx_timestamp', + service: '__hdx_service', + severity: '__hdx_severity', + body: '__hdx_body', + /** Milliseconds; only projected when a Trace source is in the selection. */ + durationMs: '__hdx_duration_ms', +} as const; + +/** + * An extra user-picked column to project alongside the canonical aliases. + * `expression` is the per-source SQL expression for the column, or null when + * the source has no such column (projected as NULL so every source returns + * the same column set). + */ +export type MultiSourceExtraColumn = { + /** Result column name (used verbatim as the quoted alias). */ + name: string; + expression: string | null; +}; + +const quoteAlias = (name: string) => `"${name.replace(/"/g, '\\"')}"`; + +/** + * Per-source semantic expression for each canonical alias. Mirrors the app's + * display helpers (`getDisplayedTimestampValueExpression`, `getEventBody`, + * `getDurationMsExpression` in packages/app/src/source.ts) — keep in sync. + */ +function multiSourceSemanticExpressions(source: TSource): { + timestamp: string; + service: string; + severity: string; + body: string; + durationMs: string; +} { + const firstTimestamp = getFirstTimestampValueExpression( + source.timestampValueExpression, + ); + + if (isLogSource(source)) { + return { + timestamp: source.displayedTimestampValueExpression || firstTimestamp, + service: source.serviceNameExpression || 'NULL', + severity: source.severityTextExpression || 'NULL', + body: source.bodyExpression || source.implicitColumnExpression || 'NULL', + durationMs: 'NULL', + }; + } + + if (isTraceSource(source)) { + return { + timestamp: source.displayedTimestampValueExpression || firstTimestamp, + service: source.serviceNameExpression || 'NULL', + severity: source.statusCodeExpression || 'NULL', + body: source.spanNameExpression || 'NULL', + // Match getDurationMsExpression: durationPrecision is the sub-second + // digit count (9 = nanoseconds), so /1e(precision-3) yields milliseconds. + durationMs: `(${source.durationExpression})/1e${(source.durationPrecision ?? 9) - 3}`, + }; + } + + // Multi-source search only supports Log and Trace sources today; other kinds + // still get a valid (if minimal) shape so a stray source can't render SQL + // that errors the whole selection. + return { + timestamp: firstTimestamp, + service: 'NULL', + severity: 'NULL', + body: 'NULL', + durationMs: 'NULL', + }; +} + +/** + * Build the canonical aliased SELECT string for one source in a multi-source + * search. Exported for tests. + */ +export function buildMultiSourceSelect( + source: TSource, + { + includeDuration = false, + extraColumns = [], + }: { + /** Project `__hdx_duration_ms` (set when any selected source is a Trace). */ + includeDuration?: boolean; + extraColumns?: MultiSourceExtraColumn[]; + } = {}, +): string { + const exprs = multiSourceSemanticExpressions(source); + + const parts = [ + `${exprs.timestamp} AS ${quoteAlias(MULTI_SOURCE_ALIASES.timestamp)}`, + `${exprs.service} AS ${quoteAlias(MULTI_SOURCE_ALIASES.service)}`, + `${exprs.severity} AS ${quoteAlias(MULTI_SOURCE_ALIASES.severity)}`, + `${exprs.body} AS ${quoteAlias(MULTI_SOURCE_ALIASES.body)}`, + ]; + if (includeDuration) { + parts.push( + `${exprs.durationMs} AS ${quoteAlias(MULTI_SOURCE_ALIASES.durationMs)}`, + ); + } + for (const col of extraColumns) { + parts.push(`${col.expression ?? 'NULL'} AS ${quoteAlias(col.name)}`); + } + + return parts.join(', '); +} + +/** + * Build the chart config for one source of a multi-source search: the standard + * `buildSearchChartConfig` assembly with the SELECT replaced by the canonical + * aliased column set, so every selected source returns the same result shape. + * + * The caller supplies `orderBy` per source (each source's own timestamp-based + * default) — a shared orderBy is meaningless across schemas, and time-window + * pagination requires the first orderBy term to be the source's timestamp. + */ +export function buildMultiSourceSearchConfig( + source: TSource, + input: Omit, + opts: { + includeDuration?: boolean; + extraColumns?: MultiSourceExtraColumn[]; + } = {}, +): SearchChartConfig { + return buildSearchChartConfig(source, { + ...input, + select: buildMultiSourceSelect(source, opts), + }); +} From 99420f82841e21fbbadc4199e79b133af47c5993 Mon Sep 17 00:00:00 2001 From: Tom Alexander Date: Wed, 12 Aug 2026 12:59:29 -0400 Subject: [PATCH 3/8] feat(app): merge and source-list primitives for cross-source search Two pure pieces the search page needs before it can span sources. The merge interleaves several already-ordered row streams into one timestamp-ordered list, bounded by a "safe frontier": the timestamp every stream has covered. Rows older than that are held back, so the timeline never shows a gap a slower stream could still fill, and the caller is told which streams are holding the frontier so only those need another page. Streams that error or are excluded stop bounding the frontier rather than freezing the list. The param resolver extends the existing single-source one to a list, deduping, capping, and reporting entries it could not resolve so one bad name doesn't sink the whole selection. Both are unit-tested; nothing calls them yet. --- .../utils/__tests__/multiSourceMerge.test.ts | 379 ++++++++++++++++++ .../src/utils/__tests__/sourceParams.test.ts | 63 +++ packages/app/src/utils/multiSourceMerge.ts | 239 +++++++++++ packages/app/src/utils/sourceParams.ts | 45 +++ 4 files changed, 726 insertions(+) create mode 100644 packages/app/src/utils/__tests__/multiSourceMerge.test.ts create mode 100644 packages/app/src/utils/multiSourceMerge.ts diff --git a/packages/app/src/utils/__tests__/multiSourceMerge.test.ts b/packages/app/src/utils/__tests__/multiSourceMerge.test.ts new file mode 100644 index 0000000000..ae77b18f9e --- /dev/null +++ b/packages/app/src/utils/__tests__/multiSourceMerge.test.ts @@ -0,0 +1,379 @@ +import { + computeFrontier, + coveredUntil, + mergeStreams, + MULTI_SOURCE_ROW_FIELDS, + StreamSnapshot, +} from '@/utils/multiSourceMerge'; + +const TS_KEY = '__hdx_timestamp'; + +const T = (iso: string) => new Date(iso); +const ms = (iso: string) => new Date(iso).getTime(); + +// Search range: 10:00 - 12:00 UTC +const DATE_RANGE: [Date, Date] = [ + T('2026-08-07T10:00:00Z'), + T('2026-08-07T12:00:00Z'), +]; + +const row = (iso: string, extra: Record = {}) => ({ + [TS_KEY]: iso, + ...extra, +}); + +const makeStream = ( + overrides: Partial & { sourceId: string }, +): StreamSnapshot => ({ + sourceName: overrides.sourceId, + rows: [], + window: null, + lastPageRowCount: null, + hasNextPage: true, + isActive: true, + dateRange: DATE_RANGE, + ...overrides, +}); + +const parseTs = (r: Record) => new Date(r[TS_KEY]).getTime(); + +describe('coveredUntil (DESC)', () => { + it('covers the whole range when the stream is fully drained', () => { + const stream = makeStream({ + sourceId: 'a', + hasNextPage: false, + window: { + startTime: T('2026-08-07T10:00:00Z'), + endTime: T('2026-08-07T12:00:00Z'), + }, + lastPageRowCount: 0, + }); + expect(coveredUntil(stream, 'DESC', parseTs)).toBe(DATE_RANGE[0].getTime()); + }); + + it('covers nothing during the initial fetch even though hasNextPage is still false', () => { + // useInfiniteQuery reports hasNextPage=false before the first page lands; + // that must not be mistaken for a drained stream. + const stream = makeStream({ sourceId: 'a', hasNextPage: false }); + expect(coveredUntil(stream, 'DESC', parseTs)).toBe(DATE_RANGE[1].getTime()); + }); + + it('covers nothing before the first page arrives', () => { + const stream = makeStream({ sourceId: 'a' }); + expect(coveredUntil(stream, 'DESC', parseTs)).toBe(DATE_RANGE[1].getTime()); + }); + + it('covers through the window start when the last page was empty', () => { + const stream = makeStream({ + sourceId: 'a', + window: { + startTime: T('2026-08-07T11:45:00Z'), + endTime: T('2026-08-07T12:00:00Z'), + }, + lastPageRowCount: 0, + }); + expect(coveredUntil(stream, 'DESC', parseTs)).toBe( + ms('2026-08-07T11:45:00Z'), + ); + }); + + it('covers only through the oldest fetched row when stopped mid-window at LIMIT', () => { + const stream = makeStream({ + sourceId: 'a', + rows: [row('2026-08-07T11:59:00Z'), row('2026-08-07T11:50:00Z')], + window: { + startTime: T('2026-08-07T11:45:00Z'), + endTime: T('2026-08-07T12:00:00Z'), + }, + lastPageRowCount: 2, + }); + expect(coveredUntil(stream, 'DESC', parseTs)).toBe( + ms('2026-08-07T11:50:00Z'), + ); + }); +}); + +describe('coveredUntil (ASC)', () => { + it('mirrors the DESC semantics from the start of the range', () => { + expect(coveredUntil(makeStream({ sourceId: 'a' }), 'ASC', parseTs)).toBe( + DATE_RANGE[0].getTime(), + ); + expect( + coveredUntil( + makeStream({ + sourceId: 'a', + hasNextPage: false, + window: { + startTime: T('2026-08-07T10:00:00Z'), + endTime: T('2026-08-07T12:00:00Z'), + }, + lastPageRowCount: 0, + }), + 'ASC', + parseTs, + ), + ).toBe(DATE_RANGE[1].getTime()); + expect( + coveredUntil( + makeStream({ + sourceId: 'a', + window: { + startTime: T('2026-08-07T10:00:00Z'), + endTime: T('2026-08-07T10:15:00Z'), + }, + lastPageRowCount: 0, + }), + 'ASC', + parseTs, + ), + ).toBe(ms('2026-08-07T10:15:00Z')); + }); +}); + +const drainedStream = (sourceId: string) => + makeStream({ + sourceId, + hasNextPage: false, + window: { + startTime: T('2026-08-07T10:00:00Z'), + endTime: T('2026-08-07T12:00:00Z'), + }, + lastPageRowCount: 0, + }); + +describe('computeFrontier', () => { + it('is the least-covered active stream (max for DESC)', () => { + const drained = drainedStream('a'); + const midWindow = makeStream({ + sourceId: 'b', + rows: [row('2026-08-07T11:50:00Z')], + window: { + startTime: T('2026-08-07T11:45:00Z'), + endTime: T('2026-08-07T12:00:00Z'), + }, + lastPageRowCount: 1, + }); + expect(computeFrontier([drained, midWindow], 'DESC', parseTs)).toBe( + ms('2026-08-07T11:50:00Z'), + ); + }); + + it('ignores inactive (errored/excluded) streams so they cannot stall the merge', () => { + const drained = drainedStream('a'); + const errored = makeStream({ sourceId: 'b', isActive: false }); + expect(computeFrontier([drained, errored], 'DESC', parseTs)).toBe( + DATE_RANGE[0].getTime(), + ); + }); + + it('is null when no stream is active', () => { + const errored = makeStream({ sourceId: 'a', isActive: false }); + expect(computeFrontier([errored], 'DESC', parseTs)).toBeNull(); + }); +}); + +describe('mergeStreams', () => { + const window0 = { + startTime: T('2026-08-07T11:45:00Z'), + endTime: T('2026-08-07T12:00:00Z'), + }; + + it('interleaves rows across streams newest-first and tags their source', () => { + const a = makeStream({ + sourceId: 'a', + sourceName: 'app logs', + rows: [row('2026-08-07T11:59:00Z'), row('2026-08-07T11:57:00Z')], + window: window0, + lastPageRowCount: 2, + hasNextPage: false, + }); + const b = makeStream({ + sourceId: 'b', + sourceName: 'traces', + rows: [row('2026-08-07T11:58:00Z')], + window: window0, + lastPageRowCount: 1, + hasNextPage: false, + }); + + const { rows } = mergeStreams([a, b], 'DESC', TS_KEY); + + expect(rows.map(r => r[TS_KEY])).toEqual([ + '2026-08-07T11:59:00Z', + '2026-08-07T11:58:00Z', + '2026-08-07T11:57:00Z', + ]); + expect(rows.map(r => r[MULTI_SOURCE_ROW_FIELDS.SOURCE_NAME])).toEqual([ + 'app logs', + 'traces', + 'app logs', + ]); + expect(rows[0][MULTI_SOURCE_ROW_FIELDS.SOURCE_ID]).toBe('a'); + }); + + it('holds back rows older than the frontier until lagging streams catch up', () => { + // Stream a is fully drained down to 10:00; stream b stopped at LIMIT with + // its oldest row at 11:50 — anything older than 11:50 from a must wait. + const a = makeStream({ + sourceId: 'a', + rows: [ + row('2026-08-07T11:55:00Z'), + row('2026-08-07T11:49:00Z'), // older than b's coverage — held back + ], + window: window0, + lastPageRowCount: 2, + hasNextPage: false, + }); + const b = makeStream({ + sourceId: 'b', + rows: [row('2026-08-07T11:50:00Z')], + window: window0, + lastPageRowCount: 1, + hasNextPage: true, + }); + + const { rows, frontier, laggingSourceIds } = mergeStreams( + [a, b], + 'DESC', + TS_KEY, + ); + + expect(frontier).toBe(ms('2026-08-07T11:50:00Z')); + expect(rows.map(r => r[TS_KEY])).toEqual([ + '2026-08-07T11:55:00Z', + '2026-08-07T11:50:00Z', + ]); + expect(laggingSourceIds).toEqual(['b']); + }); + + it('shows everything when all streams are drained', () => { + const a = makeStream({ + sourceId: 'a', + rows: [row('2026-08-07T10:05:00Z')], + window: window0, + lastPageRowCount: 1, + hasNextPage: false, + }); + const b = makeStream({ + sourceId: 'b', + rows: [row('2026-08-07T10:03:00Z')], + window: window0, + lastPageRowCount: 1, + hasNextPage: false, + }); + + const { rows, laggingSourceIds } = mergeStreams([a, b], 'DESC', TS_KEY); + + expect(rows).toHaveLength(2); + expect(laggingSourceIds).toEqual([]); + }); + + it('holds everything back while a stream has no page yet, without marking it lagging', () => { + const a = makeStream({ + sourceId: 'a', + rows: [row('2026-08-07T11:59:00Z')], + window: window0, + lastPageRowCount: 1, + hasNextPage: false, + }); + const pending = makeStream({ sourceId: 'b' }); + + const { rows, laggingSourceIds } = mergeStreams( + [a, pending], + 'DESC', + TS_KEY, + ); + + // Frontier sits at the range end until b's first page lands. + expect(rows).toEqual([]); + // b's initial fetch is already in flight — nothing to advance. + expect(laggingSourceIds).toEqual([]); + }); + + it('still shows rows from errored streams but never waits on them', () => { + const a = makeStream({ + sourceId: 'a', + rows: [row('2026-08-07T11:59:00Z')], + window: window0, + lastPageRowCount: 1, + hasNextPage: false, + }); + const errored = makeStream({ + sourceId: 'b', + rows: [row('2026-08-07T11:58:00Z')], + window: window0, + lastPageRowCount: 1, + isActive: false, + }); + + const { rows, laggingSourceIds } = mergeStreams( + [a, errored], + 'DESC', + TS_KEY, + ); + + expect(rows.map(r => r[TS_KEY])).toEqual([ + '2026-08-07T11:59:00Z', + '2026-08-07T11:58:00Z', + ]); + expect(laggingSourceIds).toEqual([]); + }); + + it('merges oldest-first with a mirrored frontier for ASC', () => { + const window0Asc = { + startTime: T('2026-08-07T10:00:00Z'), + endTime: T('2026-08-07T10:15:00Z'), + }; + const a = makeStream({ + sourceId: 'a', + rows: [ + row('2026-08-07T10:01:00Z'), + row('2026-08-07T10:20:00Z'), // beyond b's coverage — held back + ], + window: window0Asc, + lastPageRowCount: 2, + hasNextPage: false, + }); + const b = makeStream({ + sourceId: 'b', + rows: [row('2026-08-07T10:05:00Z')], + window: window0Asc, + lastPageRowCount: 1, + hasNextPage: true, + }); + + const { rows, frontier, laggingSourceIds } = mergeStreams( + [a, b], + 'ASC', + TS_KEY, + ); + + expect(frontier).toBe(ms('2026-08-07T10:05:00Z')); + expect(rows.map(r => r[TS_KEY])).toEqual([ + '2026-08-07T10:01:00Z', + '2026-08-07T10:05:00Z', + ]); + expect(laggingSourceIds).toEqual(['b']); + }); + + it('advances every stream tied at the frontier', () => { + const a = makeStream({ + sourceId: 'a', + rows: [row('2026-08-07T11:50:00Z')], + window: window0, + lastPageRowCount: 1, + hasNextPage: true, + }); + const b = makeStream({ + sourceId: 'b', + rows: [row('2026-08-07T11:50:00Z')], + window: window0, + lastPageRowCount: 1, + hasNextPage: true, + }); + + const { laggingSourceIds } = mergeStreams([a, b], 'DESC', TS_KEY); + + expect(laggingSourceIds).toEqual(['a', 'b']); + }); +}); diff --git a/packages/app/src/utils/__tests__/sourceParams.test.ts b/packages/app/src/utils/__tests__/sourceParams.test.ts index d26d25c538..c2a86598c7 100644 --- a/packages/app/src/utils/__tests__/sourceParams.test.ts +++ b/packages/app/src/utils/__tests__/sourceParams.test.ts @@ -2,6 +2,7 @@ import { SourceKind } from '@hyperdx/common-utils/dist/types'; import { resolveSourceParam, + resolveSourcesParam, SourceForParamResolution, } from '@/utils/sourceParams'; @@ -174,3 +175,65 @@ describe('resolveSourceParam', () => { }); }); }); + +describe('resolveSourcesParam', () => { + it('resolves a list by ID and by name', () => { + expect(resolveSourcesParam(['log-1', 'Traces'], SOURCES)).toEqual({ + status: 'resolved', + sources: [LOGS_1, TRACES], + unresolved: [], + }); + }); + + it('reports pending while sources are loading', () => { + expect(resolveSourcesParam(['log-1'], undefined)).toEqual({ + status: 'pending', + }); + }); + + it('resolves an empty selection without waiting on the source list', () => { + expect(resolveSourcesParam([], undefined)).toEqual({ + status: 'resolved', + sources: [], + unresolved: [], + }); + }); + + it('keeps what resolves and reports the rest, so one bad entry does not sink the selection', () => { + expect(resolveSourcesParam(['log-1', 'Nope', 'Traces'], SOURCES)).toEqual({ + status: 'resolved', + sources: [LOGS_1, TRACES], + unresolved: ['Nope'], + }); + }); + + it('reports an entry naming a source of the wrong kind', () => { + expect( + resolveSourcesParam(['log-1', 'Traces'], SOURCES, { + kinds: [SourceKind.Log], + }), + ).toEqual({ + status: 'resolved', + sources: [LOGS_1], + unresolved: ['Traces'], + }); + }); + + it('dedupes entries that resolve to the same source', () => { + expect(resolveSourcesParam(['log-1', 'log-1'], SOURCES)).toEqual({ + status: 'resolved', + sources: [LOGS_1], + unresolved: [], + }); + }); + + it('caps the selection at `max`, keeping the first entries', () => { + expect( + resolveSourcesParam(['log-1', 'Traces', 'Old Logs'], SOURCES, { max: 2 }), + ).toEqual({ + status: 'resolved', + sources: [LOGS_1, TRACES], + unresolved: [], + }); + }); +}); diff --git a/packages/app/src/utils/multiSourceMerge.ts b/packages/app/src/utils/multiSourceMerge.ts new file mode 100644 index 0000000000..ae53b730c4 --- /dev/null +++ b/packages/app/src/utils/multiSourceMerge.ts @@ -0,0 +1,239 @@ +/** + * Pure merge logic for multi-source search: k-way merges per-source result + * streams by timestamp, bounded by a "safe frontier" so the interleaved + * timeline never shows a gap another source could still fill. + * + * Every source stream paginates through the same progressive time windows + * (see utils/searchWindows.ts — windows are a pure function of the date + * range), but streams advance at different speeds: one source may be three + * windows deep while another is still mid-window at its row LIMIT. A merged + * DESC timeline is only correct down to the timestamp every stream has + * covered; rows older than that are held back until the lagging streams catch + * up. + */ + +/** Client-side fields tagged onto every merged row. Never sent to ClickHouse. */ +export const MULTI_SOURCE_ROW_FIELDS = { + SOURCE_ID: '__hdx_source_id', + SOURCE_NAME: '__hdx_source_name', + SOURCE_COLOR: '__hdx_source_color', +} as const; + +export type MergeDirection = 'ASC' | 'DESC'; + +export type StreamSnapshot = { + sourceId: string; + sourceName: string; + /** Badge/series color for this source; tagged onto rows for the table cell. */ + sourceColor?: string; + /** + * Rows fetched so far, in stream order (newest-first for DESC, + * oldest-first for ASC) — the order the windowed query produces. + */ + rows: Record[]; + /** The last fetched page's time window; null when no page has completed. */ + window: { startTime: Date; endTime: Date } | null; + /** + * Row count of the last fetched page; 0 means the window was drained, + * >0 means the stream may have stopped mid-window at its LIMIT. + * Null when no page has completed. + */ + lastPageRowCount: number | null; + hasNextPage: boolean; + /** + * Errored/excluded streams don't bound the frontier (they'd stall the merge + * forever); their already-fetched rows are still shown. + */ + isActive: boolean; + /** The full searched range, used when a stream is fully drained. */ + dateRange: [Date, Date]; +}; + +/** + * Epoch-ms timestamp T such that this stream is guaranteed to have produced + * every row it has on the already-covered side of T: + * DESC — all of the stream's rows with ts >= T are fetched; + * ASC — all of the stream's rows with ts <= T are fetched. + * + * Conservative by construction: when the stream stopped mid-window at its + * LIMIT, coverage only extends to the last row it returned, not the window + * boundary. + */ +export function coveredUntil( + stream: StreamSnapshot, + direction: MergeDirection, + parseTs: (row: Record) => number, +): number { + const [start, end] = stream.dateRange; + + if (stream.window == null || stream.lastPageRowCount == null) { + // Nothing fetched yet: no coverage at all. Checked before hasNextPage — + // useInfiniteQuery reports hasNextPage=false during the initial fetch, + // which must not read as "fully drained". + return direction === 'DESC' ? end.getTime() : start.getTime(); + } + + if (!stream.hasNextPage) { + // Fully drained: the stream covered the entire searched range. + return direction === 'DESC' ? start.getTime() : end.getTime(); + } + + if (stream.lastPageRowCount === 0) { + // The last window came back empty, so it is fully covered. + return direction === 'DESC' + ? stream.window.startTime.getTime() + : stream.window.endTime.getTime(); + } + + // Mid-window at LIMIT: covered only through the last row returned. Rows are + // in stream order, so the last row is the furthest-along one. + const lastRow = stream.rows[stream.rows.length - 1]; + if (lastRow == null) { + // Defensive: a non-zero lastPageRowCount implies rows exist. + return direction === 'DESC' + ? stream.window.endTime.getTime() + : stream.window.startTime.getTime(); + } + return parseTs(lastRow); +} + +/** + * The merge frontier: the timestamp every active stream has covered. + * DESC — rows with ts >= frontier are safe to show; ASC — ts <= frontier. + * Null when there are no active streams (nothing bounds the merge). + */ +export function computeFrontier( + streams: StreamSnapshot[], + direction: MergeDirection, + parseTs: (row: Record) => number, +): number | null { + let frontier: number | null = null; + for (const stream of streams) { + if (!stream.isActive) continue; + const covered = coveredUntil(stream, direction, parseTs); + if (frontier == null) { + frontier = covered; + } else { + frontier = + direction === 'DESC' + ? Math.max(frontier, covered) + : Math.min(frontier, covered); + } + } + return frontier; +} + +/** + * The active streams holding the frontier back that can be advanced with + * another page fetch. Streams whose initial fetch hasn't completed are not + * included — their in-flight request IS their advancement. + */ +function laggingStreams( + streams: StreamSnapshot[], + direction: MergeDirection, + parseTs: (row: Record) => number, +): StreamSnapshot[] { + const frontier = computeFrontier(streams, direction, parseTs); + if (frontier == null) return []; + return streams.filter( + stream => + stream.isActive && + stream.hasNextPage && + stream.window != null && + coveredUntil(stream, direction, parseTs) === frontier, + ); +} + +export type MergedRow = Record; + +/** + * Merge all fetched rows across streams into one timestamp-ordered list, + * tagged with their origin source, held back at the frontier. + * + * Rows from inactive (errored/excluded) streams are still included — they are + * valid data — but only active streams bound the frontier, so a dead source + * can't freeze the timeline. + */ +function mergeStreamRows( + streams: StreamSnapshot[], + direction: MergeDirection, + timestampKey: string, +): MergedRow[] { + // Timestamps repeat heavily at second precision; cache the Date parse per + // distinct raw value (same trick as ChartUtils' time-chart transform). + const tsCache = new Map(); + const parseTs = (row: Record): number => { + const raw = row[timestampKey]; + let ts = tsCache.get(raw); + if (ts === undefined) { + ts = new Date(raw).getTime(); + tsCache.set(raw, ts); + } + return ts; + }; + + const frontier = computeFrontier(streams, direction, parseTs); + + const tagged: { row: MergedRow; ts: number }[] = []; + for (const stream of streams) { + for (const row of stream.rows) { + const ts = parseTs(row); + if ( + frontier != null && + (direction === 'DESC' ? ts < frontier : ts > frontier) + ) { + continue; + } + tagged.push({ + row: { + ...row, + [MULTI_SOURCE_ROW_FIELDS.SOURCE_ID]: stream.sourceId, + [MULTI_SOURCE_ROW_FIELDS.SOURCE_NAME]: stream.sourceName, + ...(stream.sourceColor != null + ? { [MULTI_SOURCE_ROW_FIELDS.SOURCE_COLOR]: stream.sourceColor } + : {}), + }, + ts, + }); + } + } + + // Array.prototype.sort is stable, so ties keep (stream order, row order). + tagged.sort((a, b) => (direction === 'DESC' ? b.ts - a.ts : a.ts - b.ts)); + + return tagged.map(t => t.row); +} + +/** + * Convenience wrapper used by the table component: one pass producing the + * merged rows, the frontier (for the "loading up to" indicator), and which + * streams to advance on the next fetch. + */ +export function mergeStreams( + streams: StreamSnapshot[], + direction: MergeDirection, + timestampKey: string, +): { + rows: MergedRow[]; + frontier: number | null; + laggingSourceIds: string[]; +} { + const tsCache = new Map(); + const parseTs = (row: Record): number => { + const raw = row[timestampKey]; + let ts = tsCache.get(raw); + if (ts === undefined) { + ts = new Date(raw).getTime(); + tsCache.set(raw, ts); + } + return ts; + }; + + return { + rows: mergeStreamRows(streams, direction, timestampKey), + frontier: computeFrontier(streams, direction, parseTs), + laggingSourceIds: laggingStreams(streams, direction, parseTs).map( + s => s.sourceId, + ), + }; +} diff --git a/packages/app/src/utils/sourceParams.ts b/packages/app/src/utils/sourceParams.ts index 5874111c99..dc759c11b7 100644 --- a/packages/app/src/utils/sourceParams.ts +++ b/packages/app/src/utils/sourceParams.ts @@ -55,6 +55,51 @@ export type SourceParamResolution = * lowest ID, so the same link always resolves to the same source no matter what * order the API returns them in. */ +/** + * Resolve a list of source params (IDs or names) for multi-source search. + * Each element resolves with the same rules as `resolveSourceParam`; results + * are deduped by ID and capped at `max`. Elements that can't be resolved (or + * resolve to a source of the wrong kind) are reported in `unresolved` so the + * caller can warn without failing the rest of the selection. + */ +export function resolveSourcesParam( + paramValues: string[] | null | undefined, + sources: T[] | undefined, + { kinds, max }: { kinds?: SourceKind[]; max?: number } = {}, +): + | { status: 'pending' } + | { status: 'resolved'; sources: T[]; unresolved: string[] } { + if (paramValues == null || paramValues.length === 0) { + return { status: 'resolved', sources: [], unresolved: [] }; + } + if (sources == null) return { status: 'pending' }; + + const resolved: T[] = []; + const seenIds = new Set(); + const unresolved: string[] = []; + + for (const value of paramValues) { + const resolution = resolveSourceParam(value, sources, { kinds }); + if (resolution.status === 'resolved') { + if (!seenIds.has(resolution.source.id)) { + seenIds.add(resolution.source.id); + resolved.push(resolution.source); + } + } else if ( + resolution.status === 'not-found' || + resolution.status === 'wrong-kind' + ) { + unresolved.push(value); + } + } + + return { + status: 'resolved', + sources: max != null ? resolved.slice(0, max) : resolved, + unresolved, + }; +} + export function resolveSourceParam( paramValue: string | null | undefined, sources: T[] | undefined, From 5c8ed2757ab6d7c7a20349cb443bc21b49a5d854 Mon Sep 17 00:00:00 2001 From: Tom Alexander Date: Wed, 12 Aug 2026 13:01:42 -0400 Subject: [PATCH 4/8] refactor(app): extract the denoise pipeline out of the results table MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit DBSqlRowTable carried the whole denoise flow inline: mine patterns from a sample, find the ones covering more than the noise threshold, then filter the fetched rows against them. Three chained queries and their loading state, in the middle of a component that already does a lot. Pull it into useDenoisedRows plus the small summary that lists what was removed, and have the table call them. Same queries, same keys, same behavior — the component just stops owning it, and a second results table can reuse it rather than copy it. Also threads the last page's row count out of the paginated query, which a caller merging several sources needs to tell "this window is drained" from "this page stopped at its limit". --- packages/app/src/components/DBRowTable.tsx | 280 ++++++++++++------ .../app/src/components/MultiSourceBadge.tsx | 23 ++ .../app/src/hooks/useOffsetPaginatedQuery.tsx | 8 +- 3 files changed, 216 insertions(+), 95 deletions(-) create mode 100644 packages/app/src/components/MultiSourceBadge.tsx diff --git a/packages/app/src/components/DBRowTable.tsx b/packages/app/src/components/DBRowTable.tsx index 19574d849b..21834391a0 100644 --- a/packages/app/src/components/DBRowTable.tsx +++ b/packages/app/src/components/DBRowTable.tsx @@ -103,6 +103,7 @@ import { useLocalStorage, usePrevious, } from '@/utils'; +import { MULTI_SOURCE_ROW_FIELDS } from '@/utils/multiSourceMerge'; import ChartErrorState, { ChartErrorStateVariant, @@ -124,6 +125,7 @@ import { useExpandableRows, } from './ExpandableRowTable'; import LogLevel from './LogLevel'; +import { SourceBadge } from './MultiSourceBadge'; import styles from '@styles/LogTable.module.scss'; @@ -167,6 +169,7 @@ function getResolvedColumnSize( const jsType = opts.columnTypeMap.get(column)?._type; if (jsType === JSDataType.Date) return 170; if (column === opts.logLevelColumn) return 115; + if (column === MULTI_SOURCE_ROW_FIELDS.SOURCE_NAME) return 140; return 160; } @@ -615,6 +618,30 @@ export const RawLogTable = memo( const strValue = typeof value === 'string' ? value : `${value}`; + // Multi-source search tags each merged row with its origin + // source; render it as a colored badge (color assigned by the + // merge layer, consistent with the histogram series). + if (column === MULTI_SOURCE_ROW_FIELDS.SOURCE_NAME) { + return ( + + ); + } + + // Multi-source rows project NULL where a source lacks the + // field (e.g. Duration or a picked column for log rows); show a + // quiet dash instead of the literal "null". + if ( + value == null && + info.row.original[MULTI_SOURCE_ROW_FIELDS.SOURCE_ID] != null + ) { + return —; + } + if (column === logLevelColumn) { return ; } @@ -1476,6 +1503,15 @@ export function useConfigWithAdditionalSelect( }, [primaryKey, partitionKey, config, tableMetadata, columns, sourceId]); } +/** + * The user's SELECT columns, keyed by result-set name, with the row-identity + * columns the query appends (primary/partition/block keys) trimmed off. + * + * Positional rather than name-based because ClickHouse may rewrite column + * names, and the SELECT length can differ from the returned column count + * (e.g. `SELECT *`). Exported for SearchResultsTable, which resolves the same + * columns when rendering a single source's own SELECT. + */ function selectColumnMapWithoutAdditionalKeys( selectMeta: ColumnMetaType[] | undefined, additionalKeysLength: number | undefined, @@ -1503,6 +1539,144 @@ function selectColumnMapWithoutAdditionalKeys( export type DBRowTableVariant = 'default' | 'muted'; +/** + * Drop rows matching "noisy" event patterns (patterns covering more than + * DENOISE_NOISE_THRESHOLD of a sample) from an already-fetched row set. + * + * Extracted so the search results table and DBSqlRowTable share one + * implementation. Denoising is inherently single-source: it mines patterns + * from one table's body column against that source's severity expression. + */ +function useDenoisedRows({ + config, + sourceId, + processedRows, + patternColumn, + denoiseResults, + isLive, +}: { + config: BuilderChartConfigWithDateRange; + sourceId?: string; + processedRows: Record[]; + /** Result column the patterns are mined from (the last SELECT column). */ + patternColumn: string | undefined; + denoiseResults: boolean; + isLive?: boolean; +}) { + const { data: source } = useSource({ id: sourceId }); + const groupedPatterns = useGroupedPatterns({ + config, + samples: DENOISE_SAMPLE_SIZE, + bodyValueExpression: patternColumn ?? '', + severityTextExpression: + (source?.kind === SourceKind.Log + ? source.severityTextExpression + : undefined) ?? '', + totalCount: undefined, + enabled: denoiseResults, + }); + const noisyPatterns = useQuery({ + queryKey: ['noisy-patterns', config], + queryFn: async () => { + return Object.values(groupedPatterns.data).filter( + p => + p.count / (groupedPatterns.sampledRowCount ?? 1) > + DENOISE_NOISE_THRESHOLD, + ); + }, + enabled: + denoiseResults && + groupedPatterns.data != null && + Object.values(groupedPatterns.data).length > 0 && + groupedPatterns.miner != null, + }); + const noisyPatternIds = useMemo(() => { + return noisyPatterns.data?.map(p => p.id) ?? []; + }, [noisyPatterns.data]); + + const denoisedRows = useQuery({ + queryKey: [ + 'denoised-rows', + config, + denoiseResults, + // Only include processed rows if denoising is enabled + // This helps prevent the queryKey from getting extremely large + // and causing memory issues, when it's not used. + ...(denoiseResults ? [processedRows] : []), + noisyPatternIds, + patternColumn, + ], + queryFn: async () => { + if (!denoiseResults) { + return []; + } + // No noisy patterns, so no need to denoise + if (noisyPatternIds.length === 0) { + return processedRows; + } + + const matchedLogs = await groupedPatterns.miner?.matchLogs( + processedRows.map(row => row[patternColumn ?? '']), + ); + return processedRows.filter((row, i) => { + const match = matchedLogs?.[i]; + return !noisyPatternIds.includes(`${match}`); + }); + }, + placeholderData: (previousData, previousQuery) => { + // If it's the same search, but new data, return the previous data while we load + if ( + previousQuery?.queryKey?.[0] === 'denoised-rows' && + previousQuery?.queryKey?.[1] === config + ) { + return previousData; + } + return undefined; + }, + gcTime: isLive ? ms('30s') : ms('5m'), // more aggressive gc for live data, since it can end up holding lots of data + enabled: + denoiseResults && + noisyPatterns.isSuccess && + processedRows.length > 0 && + groupedPatterns.miner != null, + }); + + return { + rows: denoiseResults ? (denoisedRows.data ?? []) : processedRows, + noisyPatterns: noisyPatterns.data, + hasNoisyPatterns: noisyPatternIds.length > 0, + isFetching: + denoisedRows.isFetching || + noisyPatterns.isFetching || + groupedPatterns.isLoading, + }; +} + +/** The "Removed Noisy Event Patterns" summary shown above denoised results. */ +function DenoisedPatternsSummary({ + noisyPatterns, + hasNoisyPatterns, +}: { + noisyPatterns: { id: string; pattern: string }[] | undefined; + hasNoisyPatterns: boolean; +}) { + return ( + + + Removed Noisy Event Patterns + + + {noisyPatterns?.map(p => ( + + {p.pattern} + + ))} + {!hasNoisyPatterns && No noisy patterns found} + + + ); +} + function DBSqlRowTableComponent({ config, sourceId, @@ -1736,88 +1910,17 @@ function DBSqlRowTableComponent({ const { data: source } = useSource({ id: sourceId }); const patternColumn = columns[columns.length - 1]; - const groupedPatterns = useGroupedPatterns({ + const denoise = useDenoisedRows({ config, - samples: DENOISE_SAMPLE_SIZE, - bodyValueExpression: patternColumn ?? '', - severityTextExpression: - (source?.kind === SourceKind.Log - ? source.severityTextExpression - : undefined) ?? '', - totalCount: undefined, - enabled: denoiseResults, - }); - const noisyPatterns = useQuery({ - queryKey: ['noisy-patterns', config], - queryFn: async () => { - return Object.values(groupedPatterns.data).filter( - p => - p.count / (groupedPatterns.sampledRowCount ?? 1) > - DENOISE_NOISE_THRESHOLD, - ); - }, - enabled: - denoiseResults && - groupedPatterns.data != null && - Object.values(groupedPatterns.data).length > 0 && - groupedPatterns.miner != null, - }); - const noisyPatternIds = useMemo(() => { - return noisyPatterns.data?.map(p => p.id) ?? []; - }, [noisyPatterns.data]); - - const denoisedRows = useQuery({ - queryKey: [ - 'denoised-rows', - config, - denoiseResults, - // Only include processed rows if denoising is enabled - // This helps prevent the queryKey from getting extremely large - // and causing memory issues, when it's not used. - ...(denoiseResults ? [processedRows] : []), - noisyPatternIds, - patternColumn, - ], - queryFn: async () => { - if (!denoiseResults) { - return []; - } - // No noisy patterns, so no need to denoise - if (noisyPatternIds.length === 0) { - return processedRows; - } - - const matchedLogs = await groupedPatterns.miner?.matchLogs( - processedRows.map(row => row[patternColumn]), - ); - return processedRows.filter((row, i) => { - const match = matchedLogs?.[i]; - return !noisyPatternIds.includes(`${match}`); - }); - }, - placeholderData: (previousData, previousQuery) => { - // If it's the same search, but new data, return the previous data while we load - if ( - previousQuery?.queryKey?.[0] === 'denoised-rows' && - previousQuery?.queryKey?.[1] === config - ) { - return previousData; - } - return undefined; - }, - gcTime: isLive ? ms('30s') : ms('5m'), // more aggressive gc for live data, since it can end up holding lots of data - enabled: - denoiseResults && - noisyPatterns.isSuccess && - processedRows.length > 0 && - groupedPatterns.miner != null, + sourceId, + processedRows, + patternColumn, + denoiseResults, + isLive, }); const isLoading = denoiseResults - ? isFetching || - denoisedRows.isFetching || - noisyPatterns.isFetching || - groupedPatterns.isLoading + ? isFetching || denoise.isFetching : isFetching; const loadingDate = @@ -1828,28 +1931,17 @@ function DBSqlRowTableComponent({ return ( <> {denoiseResults && ( - - - Removed Noisy Event Patterns - - - {noisyPatterns.data?.map(p => ( - - {p.pattern} - - ))} - {noisyPatternIds.length === 0 && ( - No noisy patterns found - )} - - + )} + + {name} + + ); +} diff --git a/packages/app/src/hooks/useOffsetPaginatedQuery.tsx b/packages/app/src/hooks/useOffsetPaginatedQuery.tsx index 5a27d39b96..2dabb32720 100644 --- a/packages/app/src/hooks/useOffsetPaginatedQuery.tsx +++ b/packages/app/src/hooks/useOffsetPaginatedQuery.tsx @@ -427,7 +427,9 @@ function flattenPages(pages: TQueryFnData[]) { return pages.flatMap(p => p.data); } -function flattenData(data: TData | undefined): TQueryFnData | null { +function flattenData( + data: TData | undefined, +): (TQueryFnData & { lastPageRowCount: number }) | null { if (data == null || data.pages.length === 0) { return null; } @@ -437,6 +439,10 @@ function flattenData(data: TData | undefined): TQueryFnData | null { data: flattenPages(data.pages), chSql: data.pages[0].chSql, window: data.pages[data.pages.length - 1].window, + // Whether the last fetched page hit results distinguishes "still mid-window + // at LIMIT" from "window drained" — multi-source merge uses this to compute + // how far this stream's time coverage safely extends. + lastPageRowCount: data.pages[data.pages.length - 1].data.length, }; } From 2348d2d8c7db516672cec3d824c5e945021ae3ac Mon Sep 17 00:00:00 2001 From: Tom Alexander Date: Wed, 12 Aug 2026 13:06:06 -0400 Subject: [PATCH 5/8] feat(app): search across multiple sources, with one source as N=1 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The search page had one results table bound to one source. It now takes a list: pick up to 3 log/trace sources and their rows interleave into a single timestamp-ordered timeline, each tagged with the source it came from. One source is not a separate path — it is N=1. Its spec carries the user's own SELECT and ORDER BY, so the table renders the authored columns with no source column, sorts, denoises, and reports errors to the page exactly as before. Several sources project the canonical aliases instead, and the client merges the streams behind the safe frontier. There is no UNION: sources can live on different ClickHouse connections, each keeps its own per-table machinery (Lucene serializer, text-index detection, materialized-column rewrites, query settings), and a source that fails degrades to a status chip instead of failing the search. Selections are shareable via ?sources=. Multi-source is Lucene-only; the histogram, total count, and filters sidebar stay single-source until the next change. --- .changeset/multi-source-search.md | 15 + packages/app/src/DBSearchPage.tsx | 655 ++++++++++++++---- .../DBSearchPage.directTrace.test.tsx | 3 + packages/app/src/components/DBRowTable.tsx | 8 +- .../components/DBSearchPageFilters/hooks.ts | 5 +- .../components/DBSqlRowTableWithSidebar.tsx | 4 +- .../app/src/components/MultiSourceBadge.tsx | 13 +- .../components/MultiSourceColumnPicker.tsx | 71 ++ .../app/src/components/SearchResultsTable.tsx | 618 +++++++++++++++++ packages/app/src/defaults.ts | 8 + .../__tests__/useMultiSourceSearch.test.ts | 87 +++ .../app/src/hooks/useMultiSourceSearch.ts | 148 ++++ .../app/src/hooks/useResolvedSourcesParam.ts | 58 ++ packages/app/src/hooks/useSourceSlots.ts | 32 + 14 files changed, 1592 insertions(+), 133 deletions(-) create mode 100644 .changeset/multi-source-search.md create mode 100644 packages/app/src/components/MultiSourceColumnPicker.tsx create mode 100644 packages/app/src/components/SearchResultsTable.tsx create mode 100644 packages/app/src/hooks/__tests__/useMultiSourceSearch.test.ts create mode 100644 packages/app/src/hooks/useMultiSourceSearch.ts create mode 100644 packages/app/src/hooks/useResolvedSourcesParam.ts create mode 100644 packages/app/src/hooks/useSourceSlots.ts diff --git a/.changeset/multi-source-search.md b/.changeset/multi-source-search.md new file mode 100644 index 0000000000..9d4868f8f4 --- /dev/null +++ b/.changeset/multi-source-search.md @@ -0,0 +1,15 @@ +--- +'@hyperdx/app': minor +'@hyperdx/common-utils': minor +--- + +Search across multiple sources at once. The search page's source selector can +now expand into a multi-select (up to 3 log/trace sources): results interleave +into one timestamp-ordered timeline with a per-row source badge, normalized +columns (Timestamp, Source, Service, Level, Message, and Duration when traces +are included), a histogram stacked by source, and an add-column picker over the +union of the selected sources' columns. Each source runs its own query +pipeline — sources on different connections work, and a failing source shows a +status chip instead of failing the whole search. Multi-source mode is +Lucene-only and shareable via URL; saved searches and alerts remain +single-source for now. diff --git a/packages/app/src/DBSearchPage.tsx b/packages/app/src/DBSearchPage.tsx index cdae127d1e..c11ff3efe1 100644 --- a/packages/app/src/DBSearchPage.tsx +++ b/packages/app/src/DBSearchPage.tsx @@ -31,7 +31,10 @@ import { ColumnMeta, } from '@hyperdx/common-utils/dist/clickhouse'; import { tcFromSource } from '@hyperdx/common-utils/dist/core/metadata'; -import { buildSearchChartConfig } from '@hyperdx/common-utils/dist/core/searchChartConfig'; +import { + buildMultiSourceSearchConfig, + buildSearchChartConfig, +} from '@hyperdx/common-utils/dist/core/searchChartConfig'; import { aliasMapToWithClauses, isBrowser, @@ -95,21 +98,31 @@ import { ErrorBoundary } from '@/components/Error/ErrorBoundary'; import { FavoriteButton } from '@/components/FavoriteButton'; import ResourceTerraformPopover from '@/components/Iac/ResourceTerraformPopover'; import { InputControlled } from '@/components/InputControlled'; +import MultiSourceColumnPicker from '@/components/MultiSourceColumnPicker'; import OnboardingModal from '@/components/OnboardingModal'; import SearchWhereInput, { getStoredLanguage, } from '@/components/SearchInput/SearchWhereInput'; import SearchPageActionBar from '@/components/SearchPageActionBar'; +import SearchResultsTable from '@/components/SearchResultsTable'; import SearchTotalCountChart from '@/components/SearchTotalCountChart'; +import { SourceMultiSelectControlled } from '@/components/SourceMultiSelect'; import { TableSourceForm } from '@/components/Sources/SourceForm'; import { SourceSelectControlled } from '@/components/SourceSelect'; import { SQLInlineEditorControlled } from '@/components/SQLEditor/SQLInlineEditor'; import { Tags } from '@/components/Tags'; import { TimePicker } from '@/components/TimePicker'; import { IS_LOCAL_MODE } from '@/config'; +import { MAX_SEARCH_SOURCES } from '@/defaults'; import { useAliasMapFromChartConfig } from '@/hooks/useChartConfig'; import { useExplainQuery } from '@/hooks/useExplainQuery'; +import { + resolveExtraColumnsForSource, + unresolvedFilterColumns, + useMultiSourceColumns, +} from '@/hooks/useMultiSourceSearch'; import { useResolvedSourceParam } from '@/hooks/useResolvedSourceParam'; +import { useResolvedSourcesParam } from '@/hooks/useResolvedSourcesParam'; import { withAppNav } from '@/layout'; import { useCreateSavedSearch, @@ -133,7 +146,6 @@ import { } from '@/utils'; import ChartSQLPreview, { SQLPreview } from './components/ChartSQLPreview'; -import DBSqlRowTableWithSideBar from './components/DBSqlRowTableWithSidebar'; import PatternTable from './components/PatternTable'; import { DBSearchHeatmapChart } from './components/Search/DBSearchHeatmapChart'; import DirectTraceSidePanel from './components/Search/DirectTraceSidePanel'; @@ -181,6 +193,10 @@ const ALLOWED_SOURCE_KINDS = [SourceKind.Log, SourceKind.Trace]; const SearchConfigSchema = z.object({ select: z.string(), source: z.string(), + // Multi-source search: the full selection (2+ engages multi mode). The + // single `source` field stays the primary (= sources[0]) so every + // single-source code path keeps working unchanged. + sources: z.array(z.string()), where: z.string(), whereLanguage: z.enum(['sql', 'lucene']), orderBy: z.string(), @@ -865,6 +881,25 @@ function optimizeDefaultOrderBy( : `${orderByArr[0]} DESC`; } +/** + * Per-source default ORDER BY for multi-source search. Same resolution as + * useDefaultOrderBy minus the table sorting-key optimization (which needs a + * metadata query per source): the source's explicit orderByExpression, else + * its timestamp expression(s) DESC. Time-window pagination requires the first + * term to be the source's timestamp, which this guarantees. + */ +function multiSourceDefaultOrderBy(source: TSource): string { + const isEventSource = + source.kind === SourceKind.Log || source.kind === SourceKind.Trace; + const explicit = isEventSource ? source.orderByExpression?.trim() : undefined; + if (explicit) return explicit; + return optimizeDefaultOrderBy( + source.timestampValueExpression ?? '', + isEventSource ? source.displayedTimestampValueExpression : undefined, + undefined, + ); +} + export function useDefaultOrderBy(sourceID: string | undefined | null) { const { data: source } = useSource({ id: sourceID, @@ -895,6 +930,9 @@ function formatDroppedFiltersMessage(count: number): string { // This is outside as it needs to be a stable reference const queryStateMap = { source: parseAsString, + // JSON-encoded (not comma-separated) because source names may contain + // commas; `source` is always written alongside it as the primary. + sources: parseAsJsonEncoded(), where: parseAsStringEncoded, select: parseAsStringEncoded, whereLanguage: parseAsStringEnum<'sql' | 'lucene'>(['sql', 'lucene']), @@ -1116,6 +1154,7 @@ export function DBSearchPage() { (savedSearchId || directTraceId || rawSearchedConfig.source ? '' : defaultSourceId), + sources: searchedConfig.sources ?? [], filters: searchedConfig.filters ?? [], orderBy: searchedConfig.orderBy ?? '', }, @@ -1184,6 +1223,7 @@ export function DBSearchPage() { whereLanguage: searchedConfig?.whereLanguage ?? getStoredLanguage() ?? 'lucene', source: searchedConfig?.source ?? undefined, + sources: searchedConfig?.sources ?? [], filters: searchedConfig?.filters ?? [], orderBy: searchedConfig?.orderBy ?? '', }); @@ -1198,6 +1238,7 @@ export function DBSearchPage() { // to an existing source. const isSearchConfigEmpty = !rawSearchedConfig.source && + !rawSearchedConfig.sources?.length && !where && !select && !whereLanguage && @@ -1240,6 +1281,7 @@ export function DBSearchPage() { savedSearch, searchedConfig, rawSearchedConfig.source, + rawSearchedConfig.sources, setSearchedConfig, savedSearchId, defaultSourceId, @@ -1268,12 +1310,15 @@ export function DBSearchPage() { const onSubmit = useCallback(() => { onSearch(displayedTimeInputValue); handleSubmit( - ({ select, where, whereLanguage, source, filters, orderBy }) => { + ({ select, where, whereLanguage, source, sources, filters, orderBy }) => { setSearchedConfig({ select, where, whereLanguage, source, + // Writer discipline: only 2+ selections persist the list; a single + // selection clears it so old-style URLs stay canonical. + sources: sources.length > 1 ? sources : null, filters, orderBy, }); @@ -1301,8 +1346,123 @@ export function DBSearchPage() { [debouncedSubmit, setValue], ); + const watchedSource = useWatch({ + control, + name: 'source', + // Watch will reset when changing saved search, so we need to default to the URL + defaultValue: searchedConfig.source ?? undefined, + }); + + // --- Multi-source search: selection & schema state ------------------------ + // 2+ resolved sources in the ?sources= param engage multi mode: one + // independent query pipeline per source, merged client-side. The single + // `source` (primary) keeps every existing code path working; multi mode + // only swaps what gets rendered below. Declared before the filter-state + // hooks so they can work against the union of the selected schemas. + const { sources: searchedMultiSources } = useResolvedSourcesParam( + rawSearchedConfig.sources, + { kinds: ALLOWED_SOURCE_KINDS }, + ); + const isMultiSource = searchedMultiSources.length > 1; + // Delta/pattern analyses are per-source; multi mode pins the results view. + const effectiveAnalysisMode = isMultiSource ? 'results' : analysisMode; + // Raw SQL WHERE names concrete columns of a concrete table — reinterpreting + // it per source risks silently-wrong results, so multi mode requires Lucene. + const isMultiSourceSqlBlocked = + isMultiSource && + (searchedConfig.whereLanguage ?? getStoredLanguage() ?? 'lucene') === + 'sql' && + !!searchedConfig.where; + + // A hand-authored URL may carry only ?sources=; backfill the primary so the + // single-source machinery (form, chart config) has one. + useEffect(() => { + if (!rawSearchedConfig.source && searchedMultiSources.length > 0) { + setSearchedConfig({ source: searchedMultiSources[0].id }); + } + }, [rawSearchedConfig.source, searchedMultiSources, setSearchedConfig]); + + const watchedSources = useWatch({ control, name: 'sources' }); + const formSourceCount = watchedSources?.length ?? 0; + // The multi-select UI stays visible while the user is composing a selection + // (even before a second source is added). + const [multiPickerOpen, setMultiPickerOpen] = useState(false); + const isMultiSelectUI = multiPickerOpen || formSourceCount > 1; + const formIsMulti = formSourceCount > 1; + + const enterMultiSourceSelect = useCallback(() => { + setValue('sources', watchedSource ? [watchedSource] : []); + setMultiPickerOpen(true); + }, [setValue, watchedSource]); + + // Keep the primary `source` field in sync with the selection and re-run the + // search when the selection changes. + const prevWatchedSourcesRef = useRef(null); + useEffect(() => { + const current = watchedSources ?? []; + const prev = prevWatchedSourcesRef.current; + if (prev != null && JSON.stringify(prev) === JSON.stringify(current)) { + return; + } + prevWatchedSourcesRef.current = current; + if (prev == null) { + // Initial hydration from the URL — nothing changed. + return; + } + if (current.length > 0 && current[0] !== watchedSource) { + setValue('source', current[0]); + } + if ((prev?.length ?? 0) <= 1 && current.length > 1) { + // Entering multi mode: the single-source SELECT/ORDER BY strings don't + // translate to the canonical multi-source shape. + setValue('select', ''); + setValue('orderBy', ''); + } + debouncedSubmit(); + }, [watchedSources, watchedSource, setValue, debouncedSubmit]); + + // Collapse the picker back to the single-source select when the user + // reduces a real multi selection to one source. Keyed on the >1 → ≤1 + // transition so it can't fire in the just-opened composing state (picker + // open, one source selected, second not yet picked). + const prevFormSourceCountRef = useRef(formSourceCount); + useEffect(() => { + const prev = prevFormSourceCountRef.current; + prevFormSourceCountRef.current = formSourceCount; + if (prev > 1 && formSourceCount <= 1) { + setMultiPickerOpen(false); + } + }, [formSourceCount]); + + // Per-source top-level columns: powers the add-column picker, the + // per-source `column vs NULL` projection for user-picked extras, and + // per-source filter resolvability. Driven by the form's draft selection + // while composing (so the picker has options before the search is + // submitted), falling back to the searched selection. + const formMultiSources = useMemo( + () => + (watchedSources ?? []) + .map(id => inputSourceObjs?.find(s => s.id === id)) + .filter((s): s is TSource => s != null), + [watchedSources, inputSourceObjs], + ); + const { + columnsBySourceId, + unionColumns, + dateTimeColumns: multiDateTimeColumns, + } = useMultiSourceColumns( + formMultiSources.length > 1 + ? formMultiSources + : isMultiSource + ? searchedMultiSources + : [], + ); + // --- End multi-source selection & schema state ----------------------------- + // Top-level column names for the active source, used to quote - // filter keys that contain special characters. + // filter keys that contain special characters. In multi mode this is the + // union across the selected sources, so filter keys from any of them + // escape correctly. const { data: inputSourceColumns } = useColumns( { databaseName: inputSourceObj?.from?.databaseName ?? '', @@ -1311,20 +1471,19 @@ export function DBSearchPage() { }, { enabled: !!inputSourceObj }, ); - const knownColumns = useMemo( - () => - inputSourceColumns - ? new Set(inputSourceColumns.map(c => c.name)) - : new Set(), - [inputSourceColumns], - ); + const knownColumns = useMemo(() => { + if (isMultiSource) { + const union = new Set(); + for (const names of columnsBySourceId.values()) { + for (const name of names) union.add(name); + } + return union; + } + return inputSourceColumns + ? new Set(inputSourceColumns.map(c => c.name)) + : new Set(); + }, [inputSourceColumns, isMultiSource, columnsBySourceId]); - const watchedSource = useWatch({ - control, - name: 'source', - // Watch will reset when changing saved search, so we need to default to the URL - defaultValue: searchedConfig.source ?? undefined, - }); const prevSourceRef = useRef(watchedSource); // Set when the user switches sources via the dropdown. The follow-up // effect waits for the new source's columns to load and then drops any @@ -1347,11 +1506,21 @@ export function DBSearchPage() { const { dateTimeColumns, onResolvedColumnsChange } = useResolvedDateTimeColumns(inputSourceColumns); + // In multi mode, date/time-typed filter keys may come from any selected + // source's schema. + const effectiveDateTimeColumns = useMemo( + () => + isMultiSource && multiDateTimeColumns.size > 0 + ? new Map([...dateTimeColumns, ...multiDateTimeColumns]) + : dateTimeColumns, + [isMultiSource, dateTimeColumns, multiDateTimeColumns], + ); + const filters = useWatch({ name: 'filters', control }); const searchFilters = useSearchPageFilterState({ searchQuery: filters ?? undefined, onFilterChange: handleSetFilters, - dateTimeColumns, + dateTimeColumns: effectiveDateTimeColumns, knownColumns, }); @@ -1489,6 +1658,135 @@ export function DBSearchPage() { const { data: chartConfig, isLoading: isChartConfigLoading } = useSearchedConfigToChartConfig(chartSearchConfig, defaultSearchConfig); + // --- Multi-source search: query specs ------------------------------------- + // In multi mode the `select` param holds the extra column names picked by + // the user (the canonical columns are always projected). + const multiExtraColumnNames = useMemo( + () => + isMultiSource ? splitAndTrimWithBracket(searchedConfig.select ?? '') : [], + [isMultiSource, searchedConfig.select], + ); + + // The add-column picker edits the form's draft select (like the SELECT + // editor it replaces), then auto-submits. + const inputSelect = useWatch({ name: 'select', control }); + const multiPickerValue = useMemo( + () => (formIsMulti ? splitAndTrimWithBracket(inputSelect ?? '') : []), + [formIsMulti, inputSelect], + ); + const onMultiColumnsChange = useCallback( + (columns: string[]) => { + setValue('select', columns.join(', ')); + debouncedSubmit(); + }, + [setValue, debouncedSubmit], + ); + + // Sidebar filters apply per source. A source whose table lacks a filtered + // column can't answer the filtered search — it's excluded entirely (with a + // visible reason on its status chip) rather than silently returning rows + // that ignore the filter. + const multiSourceFilters = useMemo( + () => (isMultiSource ? (searchedConfig.filters ?? []) : []), + [isMultiSource, searchedConfig.filters], + ); + const multiDisabledReasons = useMemo(() => { + const reasons = new Map(); + if (!isMultiSource || multiSourceFilters.length === 0) return reasons; + for (const source of searchedMultiSources) { + const missing = unresolvedFilterColumns( + multiSourceFilters, + columnsBySourceId.get(source.id), + ); + if (missing.length > 0) { + reasons.set( + source.id, + `${source.name} is excluded: the active filter uses ${missing.join( + ', ', + )}, which it doesn't have`, + ); + } + } + return reasons; + }, [ + isMultiSource, + multiSourceFilters, + searchedMultiSources, + columnsBySourceId, + ]); + + // The single-source chart config, pinned to the searched time range. + const dbSqlRowTableConfig = useMemo(() => { + if (chartConfig == null) { + return undefined; + } + + return { + ...chartConfig, + dateRange: searchedTimeRange, + }; + }, [chartConfig, searchedTimeRange]); + + // The search's query plan: one spec per selected source. A single source is + // just N=1 — its spec carries the user's own SELECT/ORDER BY, so the results + // table renders exactly what the user asked for. The canonical aliases only + // come into play when there is more than one source to reconcile. + const searchStreamSpecs = useMemo(() => { + if (!isMultiSource) { + if (dbSqlRowTableConfig == null || searchedSource == null) return []; + return [{ source: searchedSource, config: dbSqlRowTableConfig }]; + } + if (isMultiSourceSqlBlocked) return []; + // Extra columns and filters both need each source's DESCRIBE (to resolve + // column-vs-NULL and filter resolvability); hold the row queries until + // they've loaded so we don't fire throwaway or erroring queries. + if ( + (multiExtraColumnNames.length > 0 || multiSourceFilters.length > 0) && + columnsBySourceId.size < searchedMultiSources.length + ) { + return []; + } + const includeDuration = searchedMultiSources.some(isTraceSource); + const where = searchedConfig.where ?? ''; + return searchedMultiSources.map(source => ({ + source, + disabledReason: multiDisabledReasons.get(source.id), + config: { + ...buildMultiSourceSearchConfig( + source, + { + where, + whereLanguage: 'lucene', + filters: multiSourceFilters, + orderBy: multiSourceDefaultOrderBy(source), + }, + { + includeDuration, + extraColumns: resolveExtraColumnsForSource( + multiExtraColumnNames, + columnsBySourceId.get(source.id), + ), + }, + ), + dateRange: searchedTimeRange, + }, + })); + }, [ + isMultiSource, + isMultiSourceSqlBlocked, + searchedMultiSources, + searchedSource, + dbSqlRowTableConfig, + searchedConfig.where, + multiExtraColumnNames, + multiSourceFilters, + multiDisabledReasons, + columnsBySourceId, + searchedTimeRange, + ]); + + // --- End multi-source search --------------------------------------------- + // query error handling const { hasQueryError, queryError } = useMemo(() => { const hasQueryError = Object.values(_queryErrors).length > 0; @@ -1650,24 +1948,16 @@ export function DBSearchPage() { setTimeout(() => setCollapseAllRows(false), 100); }, [interval, updateRelativeTimeInputValue, setIsLive]); - const dbSqlRowTableConfig = useMemo(() => { - if (chartConfig == null) { - return undefined; - } - - return { - ...chartConfig, - dateRange: searchedTimeRange, - }; - }, [chartConfig, searchedTimeRange]); - // Stable key for persisting column widths in localStorage. Scoped per saved - // search when one is loaded, else per source for ad-hoc searches. + // search when one is loaded, else per source (or source set) for ad-hoc + // searches. const columnSizeTableId = savedSearchId ? `db-search-saved-${savedSearchId}` - : searchedConfig.source - ? `db-search-source-${searchedConfig.source}` - : undefined; + : isMultiSource + ? `db-search-multi-${searchedMultiSources.map(s => s.id).join('-')}` + : searchedConfig.source + ? `db-search-source-${searchedConfig.source}` + : undefined; const displayedColumns = useMemo(() => { // `select` is typed as `string | DerivedColumn[]` upstream, but in the @@ -1969,6 +2259,28 @@ export function DBSearchPage() { ], ); + // Multi-source rows span schemas, so the single-source column toggles are + // omitted — the side panel hides them. Property-add-to-filter IS wired: + // filters resolve per source, and a source that lacks the column is + // excluded with a visible reason. Passing no `source` is required: with a + // null context source, deriveRowSidePanelContextForSource treats every row + // as same-source, which is exactly the cross-source semantics filters now + // have. + const multiRowTableContext = useMemo( + () => ({ + onPropertyAddClick: searchFilters.setFilterValue, + generateSearchUrl, + isChildModalOpen: isDrawerChildModalOpen, + setChildModalOpen: setDrawerChildModalOpen, + }), + [ + searchFilters.setFilterValue, + generateSearchUrl, + isDrawerChildModalOpen, + setDrawerChildModalOpen, + ], + ); + const inputSourceTableConnection = useMemo( () => tcFromSource(inputSourceObj), [inputSourceObj], @@ -2220,22 +2532,49 @@ export function DBSearchPage() { > {/* */} - setIsSourceSchemaPreviewOpen(true)} - isSchemaPreviewEnabled={isSourceSchemaPreviewEnabled( - inputSourceObj, - )} - allowedSourceKinds={ALLOWED_SOURCE_KINDS} - data-testid="source-selector" - style={{ minWidth: 150 }} - /> + {isMultiSelectUI ? ( + + ) : ( + <> + setIsSourceSchemaPreviewOpen(true)} + isSchemaPreviewEnabled={isSourceSchemaPreviewEnabled( + inputSourceObj, + )} + allowedSourceKinds={ALLOWED_SOURCE_KINDS} + data-testid="source-selector" + style={{ minWidth: 150 }} + /> + + + + + + + )} setIsSourceSchemaPreviewOpen(false)} /> - - - - + {formIsMulti ? ( + + ) : ( + + )} + {!formIsMulti && ( + + + + )} <> {!savedSearchId ? ( - + + ) : ( + + )} @@ -2346,7 +2710,7 @@ export function DBSearchPage() { setInputValue={setDisplayedTimeInputValue} onSearch={onTimePickerSearch} onRelativeSearch={onTimePickerRelativeSearch} - showLive={analysisMode === 'results'} + showLive={effectiveAnalysisMode === 'results'} isLiveMode={isLive} // Default to relative time mode if the user has made changes to interval and reloaded. defaultRelativeTimeMode={ @@ -2381,7 +2745,7 @@ export function DBSearchPage() { @@ -2425,7 +2789,7 @@ export function DBSearchPage() { height: '100%', }} > - {!isFilterSidebarCollapsed && ( + {!isFilterSidebarCollapsed && !isMultiSource && ( )} - {analysisMode === 'pattern' && + {effectiveAnalysisMode === 'pattern' && histogramTimeChartConfig != null && ( @@ -2523,7 +2887,7 @@ export function DBSearchPage() { )} - {analysisMode === 'delta' && + {effectiveAnalysisMode === 'delta' && searchedSource != null && isTraceSource(searchedSource) && ( )} - {analysisMode === 'results' && ( + {effectiveAnalysisMode === 'results' && isMultiSource && ( + + {isMultiSourceSqlBlocked ? ( + + + SQL search isn't supported across multiple sources + + + A SQL WHERE clause references the columns of one + specific table. Switch the search language to Lucene to + search across sources, or go back to a single source. + + + ) : ( + <> + {/* The histogram, total count, and filters sidebar come + with the next change; searching several sources + returns the merged results table on its own. */} + + + + + )} + + )} + {effectiveAnalysisMode === 'results' && !isMultiSource && ( {chartConfig && histogramTimeChartConfig && ( <> @@ -2725,32 +3134,26 @@ export function DBSearchPage() { px="sm" data-testid="search-results-panel" > - {chartConfig && - searchedConfig.source && - dbSqlRowTableConfig && ( - - )} + )} diff --git a/packages/app/src/__tests__/DBSearchPage.directTrace.test.tsx b/packages/app/src/__tests__/DBSearchPage.directTrace.test.tsx index d3f10d02a5..330a024fbd 100644 --- a/packages/app/src/__tests__/DBSearchPage.directTrace.test.tsx +++ b/packages/app/src/__tests__/DBSearchPage.directTrace.test.tsx @@ -211,6 +211,9 @@ jest.mock('../components/ChartSQLPreview', () => ({ SQLPreview: () =>
, })); jest.mock('../components/DBSqlRowTableWithSidebar', () => () =>
); +// Multi-source components pull in DBRowSidePanel (and its deep import graph), +// which this test isolates away just like DBSqlRowTableWithSidebar above. +jest.mock('../components/SearchResultsTable', () => () =>
); jest.mock('../components/PatternTable', () => () =>
); jest.mock('../components/Search/DBSearchHeatmapChart', () => ({ DBSearchHeatmapChart: () =>
, diff --git a/packages/app/src/components/DBRowTable.tsx b/packages/app/src/components/DBRowTable.tsx index 21834391a0..59bdf5398d 100644 --- a/packages/app/src/components/DBRowTable.tsx +++ b/packages/app/src/components/DBRowTable.tsx @@ -1419,7 +1419,7 @@ export function appendSelectWithAdditionalKeys( } } -function getSelectLength(select: SelectList): number { +export function getSelectLength(select: SelectList): number { if (typeof select === 'string') { return select.split(',').filter(s => s.trim().length > 0).length; } else { @@ -1512,7 +1512,7 @@ export function useConfigWithAdditionalSelect( * (e.g. `SELECT *`). Exported for SearchResultsTable, which resolves the same * columns when rendering a single source's own SELECT. */ -function selectColumnMapWithoutAdditionalKeys( +export function selectColumnMapWithoutAdditionalKeys( selectMeta: ColumnMetaType[] | undefined, additionalKeysLength: number | undefined, ): Map< @@ -1547,7 +1547,7 @@ export type DBRowTableVariant = 'default' | 'muted'; * implementation. Denoising is inherently single-source: it mines patterns * from one table's body column against that source's severity expression. */ -function useDenoisedRows({ +export function useDenoisedRows({ config, sourceId, processedRows, @@ -1653,7 +1653,7 @@ function useDenoisedRows({ } /** The "Removed Noisy Event Patterns" summary shown above denoised results. */ -function DenoisedPatternsSummary({ +export function DenoisedPatternsSummary({ noisyPatterns, hasNoisyPatterns, }: { diff --git a/packages/app/src/components/DBSearchPageFilters/hooks.ts b/packages/app/src/components/DBSearchPageFilters/hooks.ts index 8c4b924bb8..a9d676102a 100644 --- a/packages/app/src/components/DBSearchPageFilters/hooks.ts +++ b/packages/app/src/components/DBSearchPageFilters/hooks.ts @@ -254,6 +254,7 @@ export function useFetchFacets({ filterState, showMoreFields, disableValues, + enabled = true, }: { chartConfig: BuilderChartConfigWithDateRange; sourceId: string | null; @@ -262,6 +263,8 @@ export function useFetchFacets({ filterState?: FilterState; showMoreFields?: boolean; disableValues?: boolean; + /** Disable all data fetching (e.g. an unused multi-source hook slot). */ + enabled?: boolean; }) { const facetsQuery = useFacets({ chartConfig, @@ -270,7 +273,7 @@ export function useFetchFacets({ dateRange, filterState, showMoreFields, - enabled: true, + enabled, disableValues, }); diff --git a/packages/app/src/components/DBSqlRowTableWithSidebar.tsx b/packages/app/src/components/DBSqlRowTableWithSidebar.tsx index bd5824e372..b500296ede 100644 --- a/packages/app/src/components/DBSqlRowTableWithSidebar.tsx +++ b/packages/app/src/components/DBSqlRowTableWithSidebar.tsx @@ -157,7 +157,9 @@ enum InlineTab { ColumnValues = 'columnValues', } -function RowOverviewPanelWrapper({ +// Exported for MultiSourceRowTable, which renders the same expanded-row +// overview but resolves the source per row instead of once per table. +export function RowOverviewPanelWrapper({ source, rowId, aliasWith, diff --git a/packages/app/src/components/MultiSourceBadge.tsx b/packages/app/src/components/MultiSourceBadge.tsx index b536e9219b..7d9d75798f 100644 --- a/packages/app/src/components/MultiSourceBadge.tsx +++ b/packages/app/src/components/MultiSourceBadge.tsx @@ -1,4 +1,15 @@ -import React from 'react'; +import { COLORS } from '@/utils'; + +/** + * Stable color for the Nth selected source in a multi-source search. Indexed + * by position in the selection (not hashed) so the ≤MAX_SEARCH_SOURCES badges + * never collide; the same assignment is used by the results table badge, the + * histogram series, and the per-source status chips so a source reads as one + * color everywhere on the page. + */ +export function getMultiSourceColor(index: number): string { + return COLORS[index % COLORS.length]; +} /** Colored-dot source label used in the merged results table. */ export function SourceBadge({ name, color }: { name: string; color?: string }) { diff --git a/packages/app/src/components/MultiSourceColumnPicker.tsx b/packages/app/src/components/MultiSourceColumnPicker.tsx new file mode 100644 index 0000000000..55230dd479 --- /dev/null +++ b/packages/app/src/components/MultiSourceColumnPicker.tsx @@ -0,0 +1,71 @@ +import { useCallback, useMemo } from 'react'; +import { Group, MultiSelect, Text } from '@mantine/core'; + +import { MultiSourceColumnOption } from '@/hooks/useMultiSourceSearch'; + +/** + * Multi-source replacement for the free-text SELECT editor: pick extra + * columns from the union of the selected sources' top-level columns. Columns + * missing from a source render as blank cells for that source's rows. + */ +export default function MultiSourceColumnPicker({ + unionColumns, + totalSources, + value, + onChange, +}: { + unionColumns: MultiSourceColumnOption[]; + totalSources: number; + /** Currently selected extra column names. */ + value: string[]; + onChange: (columns: string[]) => void; +}) { + const availabilityByName = useMemo( + () => new Map(unionColumns.map(c => [c.name, c.availableCount])), + [unionColumns], + ); + + const data = useMemo( + () => + unionColumns.map(c => ({ + value: c.name, + label: c.name, + })), + [unionColumns], + ); + + const renderOption = useCallback( + ({ option }: { option: { value: string; label: string } }) => { + const available = availabilityByName.get(option.value) ?? 0; + return ( + + + {option.label} + + {available < totalSources && ( + + {available}/{totalSources} sources + + )} + + ); + }, + [availabilityByName, totalSources], + ); + + return ( + + ); +} diff --git a/packages/app/src/components/SearchResultsTable.tsx b/packages/app/src/components/SearchResultsTable.tsx new file mode 100644 index 0000000000..4129828b92 --- /dev/null +++ b/packages/app/src/components/SearchResultsTable.tsx @@ -0,0 +1,618 @@ +import { useCallback, useEffect, useMemo, useState } from 'react'; +import { useQueryState } from 'nuqs'; +import { + chSqlToAliasMap, + ClickHouseQueryError, + ColumnMetaType, + convertCHDataTypeToJSType, + isJSDataTypeJSONStringifiable, + JSDataType, +} from '@hyperdx/common-utils/dist/clickhouse'; +import { MULTI_SOURCE_ALIASES } from '@hyperdx/common-utils/dist/core/searchChartConfig'; +import { + BuilderChartConfigWithDateRange, + SourceKind, + TSource, +} from '@hyperdx/common-utils/dist/types'; +import { Flex, Group, Loader, Text, Tooltip } from '@mantine/core'; +import { IconAlertTriangle, IconFilterOff } from '@tabler/icons-react'; +import { SortingState } from '@tanstack/react-table'; + +import api from '@/api'; +import { searchChartConfigDefaults } from '@/defaults'; +import useOffsetPaginatedQuery from '@/hooks/useOffsetPaginatedQuery'; +import useRowWhere, { RowWhereResult, WithClause } from '@/hooks/useRowWhere'; +import { useMultiSourceSlots } from '@/hooks/useSourceSlots'; +import { + mergeStreams, + MULTI_SOURCE_ROW_FIELDS, + StreamSnapshot, +} from '@/utils/multiSourceMerge'; +import { parseAsStringEncoded } from '@/utils/queryParsers'; + +import ChartErrorState from './charts/ChartErrorState'; +import DBRowSidePanel, { + RowSidePanelContext, + RowSidePanelContextProps, +} from './DBRowSidePanel'; +import { + DenoisedPatternsSummary, + getSelectLength, + RawLogTable, + selectColumnMapWithoutAdditionalKeys, + useConfigWithAdditionalSelect, + useDenoisedRows, +} from './DBRowTable'; +import { RowOverviewPanelWrapper } from './DBSqlRowTableWithSidebar'; +import { getMultiSourceColor, SourceBadge } from './MultiSourceBadge'; + +/** + * One selected source plus its fully-built chart config. + * + * With a single source the config carries that source's own SELECT (the user + * authored it); with several, each config projects the canonical + * MULTI_SOURCE_ALIASES so the merged rows share one shape. + */ +export type SearchStreamSpec = { + source: TSource; + config: BuilderChartConfigWithDateRange; + /** + * When set, the source doesn't run at all (e.g. an active filter references + * a column its table lacks); shown on the source's status chip. + */ + disabledReason?: string; +}; + +// Placeholder config for unused hook slots. The metadata hooks inside +// useConfigWithAdditionalSelect self-disable on empty table names, and the +// paginated query slot is explicitly disabled, so this never reaches +// ClickHouse. +const STUB_CONFIG: BuilderChartConfigWithDateRange = { + connection: '', + from: { databaseName: '', tableName: '' }, + timestampValueExpression: '', + select: '', + where: '', + whereLanguage: 'sql', + dateRange: [new Date(0), new Date(0)], +}; + +const EMPTY_CHSQL = { sql: '', params: {} }; +const EMPTY_EXTRA_COLUMNS: string[] = []; + +type SourceStream = { + spec: SearchStreamSpec | undefined; + data: ReturnType['data']; + fetchNextPage: ReturnType['fetchNextPage']; + hasNextPage: boolean; + isFetching: boolean; + isError: boolean; + error: Error | ClickHouseQueryError | null; + getRowWhere: (row: Record) => RowWhereResult; + /** Row-identity columns appended to the SELECT, trimmed off for display. */ + additionalKeysLength: number | undefined; +}; + +/** + * One source's independent query pipeline: the same + * defaults → additional-key SELECT merge → windowed offset pagination → + * row-WHERE machinery as the single-source DBSqlRowTable, packaged as a + * useMultiSourceSlots slot hook. Unused slots get a stub config and stay + * disabled. + */ +function useSourceStream( + spec: SearchStreamSpec | undefined, + { + enabled, + isLive, + enableSmallFirstWindow, + queryKeyPrefix, + }: { + enabled: boolean; + isLive: boolean; + enableSmallFirstWindow?: boolean; + queryKeyPrefix?: string; + }, +): SourceStream { + const { data: me } = api.useMe(); + + const configWithDefaults = useMemo( + () => ({ + ...searchChartConfigDefaults(me?.team), + ...(spec?.config ?? STUB_CONFIG), + }), + [me, spec?.config], + ); + + const mergedConfig = useConfigWithAdditionalSelect( + configWithDefaults, + spec?.source.id, + ); + + const { data, fetchNextPage, hasNextPage, isFetching, isError, error } = + useOffsetPaginatedQuery(mergedConfig ?? configWithDefaults, { + enabled: + enabled && + spec != null && + spec.disabledReason == null && + mergedConfig != null && + // An empty SELECT renders invalid SQL; wait for one to resolve. + getSelectLength(spec.config.select) > 0, + isLive, + queryKeyPrefix, + enableSmallFirstWindow, + }); + + const aliasMap = useMemo(() => { + const map = chSqlToAliasMap(data?.chSql ?? EMPTY_CHSQL); + // NULL-literal projections (`NULL AS "__hdx_duration_ms"` where a source + // lacks the field) are dropped by the SQL alias parser. Backfill them so + // the row-WHERE clause emits `isNull(NULL)` rather than referencing the + // alias as a (nonexistent) table column. ClickHouse reports NULL literals + // as Nullable(Nothing). + for (const col of data?.meta ?? []) { + if (map[col.name] == null && col.type === 'Nullable(Nothing)') { + map[col.name] = 'NULL'; + } + } + return map; + }, [data]); + + const getRowWhere = useRowWhere({ + meta: data?.meta, + aliasMap, + primaryKeyColumns: mergedConfig?.rowKeyColumns, + }); + + // Stable identity per content change, so downstream merge memos don't + // recompute (and re-sort every fetched row) on unrelated parent renders. + return useMemo( + () => ({ + spec, + data, + fetchNextPage, + hasNextPage: hasNextPage ?? false, + isFetching, + isError, + error: error ?? null, + getRowWhere, + additionalKeysLength: mergedConfig?.additionalKeysLength, + }), + [ + spec, + data, + fetchNextPage, + hasNextPage, + isFetching, + isError, + error, + getRowWhere, + mergedConfig?.additionalKeysLength, + ], + ); +} + +const COLUMN_NAME_MAP: Record = { + [MULTI_SOURCE_ALIASES.timestamp]: 'Timestamp', + [MULTI_SOURCE_ROW_FIELDS.SOURCE_NAME]: 'Source', + [MULTI_SOURCE_ALIASES.service]: 'Service', + [MULTI_SOURCE_ALIASES.severity]: 'Level', + [MULTI_SOURCE_ALIASES.durationMs]: 'Duration (ms)', + [MULTI_SOURCE_ALIASES.body]: 'Message', +}; + +function StreamStatusChips({ streams }: { streams: SourceStream[] }) { + return ( + + {streams.map((stream, i) => { + if (stream.spec == null) return null; + const name = stream.spec.source.name; + const disabledReason = stream.spec.disabledReason; + return ( + + + {stream.isFetching && } + {disabledReason != null && ( + + + + + + )} + {stream.isError && ( + + + + + + )} + + ); + })} + + ); +} + +export default function SearchResultsTable({ + sources: specs, + isLive, + enabled = true, + extraColumnNames = EMPTY_EXTRA_COLUMNS, + denoiseResults = false, + sortOrder, + onSortingChange, + onError, + onResolvedColumnsChange, + onScroll, + onSidebarOpen, + onExpandedRowsChange, + collapseAllRows, + enableSmallFirstWindow, + tableId, + context, + keepOpenSelector, + // Row queries are keyed separately from the page's chart/count queries, so + // "is the search fetching?" (live-tail pause, latency telemetry) keeps + // measuring the same thing it always has. + queryKeyPrefix = 'dbSqlRowTable', +}: { + /** 1..MAX_SEARCH_SOURCES selected sources with their built configs. */ + sources: SearchStreamSpec[]; + isLive: boolean; + enabled?: boolean; + /** User-picked extra columns projected into every source's SELECT (N>1). */ + extraColumnNames?: string[]; + /** Drop noisy event patterns from the results (single source only). */ + denoiseResults?: boolean; + /** Current sort, for the single-source case where sorting is supported. */ + sortOrder?: SortingState; + onSortingChange?: (v: SortingState | null) => void; + /** + * Surface a query failure to the page. Only called with a single source — + * with several, a failing source is isolated to its own status chip rather + * than failing the whole search. + */ + onError?: (error: Error | ClickHouseQueryError) => void; + onResolvedColumnsChange?: (meta: ColumnMetaType[]) => void; + onScroll?: (scrollTop: number) => void; + onSidebarOpen?: (rowId: string) => void; + onExpandedRowsChange?: (hasExpandedRows: boolean) => void; + collapseAllRows?: boolean; + enableSmallFirstWindow?: boolean; + tableId?: string; + context?: RowSidePanelContextProps; + keepOpenSelector?: string; + queryKeyPrefix?: string; +}) { + const slots = useMultiSourceSlots(specs, useSourceStream, { + enabled, + isLive, + enableSmallFirstWindow, + queryKeyPrefix, + }); + + const streams = useMemo( + () => + slots.filter( + (s): s is SourceStream & { spec: SearchStreamSpec } => s.spec != null, + ), + [slots], + ); + + // With one source the table shows that source's own SELECT, sorts, and + // denoises — everything the single-source search has always done. The + // canonical aliases, source badges, and cross-source merge only come into + // play once a second source is selected. + const isSingleSource = specs.length === 1; + const singleStream = isSingleSource ? streams[0] : undefined; + + const snapshots: StreamSnapshot[] = useMemo( + () => + streams.map((stream, i) => ({ + sourceId: stream.spec.source.id, + sourceName: stream.spec.source.name, + sourceColor: getMultiSourceColor(i), + rows: stream.data?.data ?? [], + window: stream.data?.window ?? null, + lastPageRowCount: stream.data?.lastPageRowCount ?? null, + hasNextPage: stream.hasNextPage, + isActive: !stream.isError && stream.spec.disabledReason == null, + dateRange: stream.spec.config.dateRange, + })), + [streams], + ); + + // One source needs no merge: its rows already arrive timestamp-ordered from + // its own ORDER BY, and there is no other stream to hold a frontier against. + const merged = useMemo( + () => + isSingleSource + ? null + : mergeStreams(snapshots, 'DESC', MULTI_SOURCE_ALIASES.timestamp), + [isSingleSource, snapshots], + ); + + const columnTypeMap = useMemo(() => { + if (singleStream != null) { + // The user's SELECT columns, positionally trimmed of the row-identity + // columns the query appends (same resolution as DBSqlRowTable). + return selectColumnMapWithoutAdditionalKeys( + singleStream.data?.meta, + singleStream.additionalKeysLength, + ); + } + // Merge column meta across streams by canonical alias name, preferring a + // resolved type over the Nullable(Nothing) a `NULL AS "alias"` projection + // reports. + const map = new Map(); + for (const stream of streams) { + for (const col of stream.data?.meta ?? []) { + const jsType = convertCHDataTypeToJSType(col.type); + const existing = map.get(col.name); + if (existing == null || existing._type == null) { + map.set(col.name, { _type: jsType }); + } + } + } + map.set(MULTI_SOURCE_ROW_FIELDS.SOURCE_NAME, { + _type: JSDataType.String, + }); + return map; + }, [streams, singleStream]); + + const includeDuration = specs.some(s => s.source.kind === SourceKind.Trace); + + const displayedColumns = useMemo(() => { + if (isSingleSource) { + return Array.from(columnTypeMap.keys()); + } + return [ + MULTI_SOURCE_ALIASES.timestamp, + MULTI_SOURCE_ROW_FIELDS.SOURCE_NAME, + MULTI_SOURCE_ALIASES.service, + MULTI_SOURCE_ALIASES.severity, + ...(includeDuration ? [MULTI_SOURCE_ALIASES.durationMs] : []), + ...extraColumnNames, + MULTI_SOURCE_ALIASES.body, + ]; + }, [isSingleSource, columnTypeMap, includeDuration, extraColumnNames]); + + // Stringify object-typed cells (Map/Array/JSON) the same way DBSqlRowTable + // does — both for display and because useRowWhere expects the stringified + // form when rebuilding a row WHERE clause. + const rows = useMemo(() => { + const baseRows = singleStream + ? (singleStream.data?.data ?? []) + : (merged?.rows ?? []); + const objectColumns = [...columnTypeMap.entries()] + .filter(([, v]) => isJSDataTypeJSONStringifiable(v._type)) + .map(([name]) => name); + if (objectColumns.length === 0) { + return baseRows; + } + return baseRows.map(row => { + const newRow = { ...row }; + for (const col of objectColumns) { + if (!(col in newRow) || newRow[col] == null) continue; + if (columnTypeMap.get(col)?._type === JSDataType.JSON) { + newRow[col] = JSON.stringify(newRow[col]).replace(/\//g, '\\/'); + } else { + newRow[col] = JSON.stringify(newRow[col]); + } + } + return newRow; + }); + }, [singleStream, merged?.rows, columnTypeMap]); + + const patternColumn = displayedColumns[displayedColumns.length - 1]; + const denoise = useDenoisedRows({ + config: singleStream?.spec.config ?? STUB_CONFIG, + sourceId: singleStream?.spec.source.id, + processedRows: rows, + patternColumn, + // Denoising mines patterns from one table's body column; it has no + // cross-source meaning, so it only runs with a single source. + denoiseResults: denoiseResults && isSingleSource, + isLive, + }); + + // Row identity dispatches to the row's own stream: each stream has its own + // result meta / alias map / primary-key columns. The client-side source tags + // are stripped first — they aren't real columns. + const generateRowId = useCallback( + (row: Record): RowWhereResult => { + if (singleStream != null) { + return singleStream.getRowWhere(row); + } + const { + [MULTI_SOURCE_ROW_FIELDS.SOURCE_ID]: sourceId, + [MULTI_SOURCE_ROW_FIELDS.SOURCE_NAME]: _name, + [MULTI_SOURCE_ROW_FIELDS.SOURCE_COLOR]: _color, + ...dbRow + } = row; + const stream = streams.find(s => s.spec.source.id === sourceId); + if (stream == null) { + return { where: '', aliasWith: [] }; + } + return stream.getRowWhere(dbRow); + }, + [streams, singleStream], + ); + + // Advance only the stream(s) holding the frontier back; the leaders keep + // their fetched-but-held rows until the laggards catch up. + const fetchNextPage = useCallback(() => { + if (singleStream != null) { + singleStream.fetchNextPage({ cancelRefetch: false }); + return; + } + for (const sourceId of merged?.laggingSourceIds ?? []) { + const stream = streams.find(s => s.spec.source.id === sourceId); + stream?.fetchNextPage({ cancelRefetch: false }); + } + }, [singleStream, merged?.laggingSourceIds, streams]); + + const hasNextPage = streams.some(s => !s.isError && s.hasNextPage); + const isFetching = streams.some(s => s.isFetching); + const isLoading = denoiseResults + ? isFetching || denoise.isFetching + : isFetching; + const allFailed = streams.length > 0 && streams.every(s => s.isError); + const firstError = streams.find(s => s.error != null)?.error ?? undefined; + + // A single source's failure is the whole search's failure, so the page owns + // the error UI (and drops out of live tail), exactly as before. + useEffect(() => { + if (singleStream?.isError && singleStream.error != null) { + onError?.(singleStream.error); + } + }, [singleStream?.isError, singleStream?.error, onError]); + + const singleMeta = singleStream?.data?.meta; + useEffect(() => { + if (singleMeta != null && singleMeta.length > 0) { + onResolvedColumnsChange?.(singleMeta); + } + }, [singleMeta, onResolvedColumnsChange]); + + // Side panel wiring — the same URL-param contract as the legacy table, + // except the panel's source comes from the clicked row rather than being + // fixed for the page. + const [rowId, setRowId] = useQueryState('rowWhere', parseAsStringEncoded); + const [rowSource, setRowSource] = useQueryState('rowSource'); + const [aliasWith, setAliasWith] = useState([]); + + const onRowDetailsClick = useCallback( + (row: Record) => { + const rowWhere = generateRowId(row); + if (!rowWhere.where) return; + setRowId(rowWhere.where); + setAliasWith(rowWhere.aliasWith); + setRowSource( + row[MULTI_SOURCE_ROW_FIELDS.SOURCE_ID] ?? + singleStream?.spec.source.id ?? + null, + ); + onSidebarOpen?.(rowWhere.where); + }, + [generateRowId, setRowId, setRowSource, onSidebarOpen, singleStream], + ); + + const onCloseSidebar = useCallback(() => { + setRowId(null); + setRowSource(null); + }, [setRowId, setRowSource]); + + const sourceForRow = useCallback( + (id: unknown) => + specs.find(s => s.source.id === id)?.source ?? + // Links predating the rowSource param (and every single-source link) + // carry only rowWhere; there is exactly one source it can belong to. + (isSingleSource ? specs[0]?.source : undefined), + [specs, isSingleSource], + ); + + const panelSource = useMemo( + () => sourceForRow(rowSource), + [sourceForRow, rowSource], + ); + + const renderRowDetails = useCallback( + (r: { id: string; aliasWith?: WithClause[]; [key: string]: unknown }) => { + const source = sourceForRow(r[MULTI_SOURCE_ROW_FIELDS.SOURCE_ID]); + if (!source) { + return
Loading...
; + } + return ( + + ); + }, + [sourceForRow], + ); + + const loadingDate = singleStream + ? singleStream.data?.window?.direction === 'ASC' + ? singleStream.data?.window?.endTime + : singleStream.data?.window?.startTime + : merged?.frontier != null && hasNextPage + ? new Date(merged.frontier) + : undefined; + + const firstConfig = streams[0]?.spec.config; + + return ( + + {panelSource != null && ( + + )} + + {/* One source needs no legend: every row came from it. */} + {!isSingleSource && } + {denoiseResults && isSingleSource && ( + + )} + {allFailed && !isSingleSource ? ( + + ) : ( + + )} + + + ); +} diff --git a/packages/app/src/defaults.ts b/packages/app/src/defaults.ts index 5894d8b78d..5b67f68216 100644 --- a/packages/app/src/defaults.ts +++ b/packages/app/src/defaults.ts @@ -2,6 +2,14 @@ import type { BuilderChartConfigWithDateRange } from '@hyperdx/common-utils/dist // Limit defaults export const DEFAULT_SEARCH_ROW_LIMIT = 200; + +// Ceiling on how many sources one search can span. Cost scales linearly with +// the selection: each source runs its own result stream plus histogram/count +// aggregates (~3 ClickHouse queries per source per refresh, re-fired every +// live-tail tick), so 3 keeps the worst case bounded while covering the +// common "app logs + infra logs + traces" setups. Also the hook-slot count in +// useMultiSourceSlots — raising it means adding a slot there too. +export const MAX_SEARCH_SOURCES = 3; export const DEFAULT_QUERY_TIMEOUT = 60; // max_execution_time, seconds export const DEFAULT_FILTER_KEYS_FETCH_LIMIT = 100; export const DEFAULT_SERIES_LIMIT = 100; diff --git a/packages/app/src/hooks/__tests__/useMultiSourceSearch.test.ts b/packages/app/src/hooks/__tests__/useMultiSourceSearch.test.ts new file mode 100644 index 0000000000..58be8d33d2 --- /dev/null +++ b/packages/app/src/hooks/__tests__/useMultiSourceSearch.test.ts @@ -0,0 +1,87 @@ +import { Filter } from '@hyperdx/common-utils/dist/types'; + +import { + filterRootColumn, + resolveExtraColumnsForSource, + unresolvedFilterColumns, +} from '@/hooks/useMultiSourceSearch'; + +describe('filterRootColumn', () => { + it('extracts a plain column reference', () => { + const filter: Filter = { + type: 'sql_ast', + operator: '=', + left: 'ServiceName', + right: "'cart'", + }; + expect(filterRootColumn(filter)).toBe('ServiceName'); + }); + + it('extracts the root of a map subscript', () => { + const filter: Filter = { + type: 'sql_ast', + operator: '=', + left: "LogAttributes['level']", + right: "'error'", + }; + expect(filterRootColumn(filter)).toBe('LogAttributes'); + }); + + it('extracts a backticked identifier', () => { + const filter: Filter = { + type: 'sql_ast', + operator: '=', + left: '`weird-col`', + right: "'x'", + }; + expect(filterRootColumn(filter)).toBe('weird-col'); + }); + + it('returns null for raw sql and lucene filters', () => { + expect( + filterRootColumn({ type: 'sql', condition: "Foo = 'bar'" }), + ).toBeNull(); + expect( + filterRootColumn({ type: 'lucene', condition: 'foo:bar' }), + ).toBeNull(); + }); +}); + +describe('unresolvedFilterColumns', () => { + const filters: Filter[] = [ + { type: 'sql_ast', operator: '=', left: 'ServiceName', right: "'cart'" }, + { type: 'sql_ast', operator: '=', left: 'StatusCode', right: "'Unset'" }, + { type: 'sql', condition: 'anything' }, + ]; + + it('reports columns the source lacks', () => { + expect( + unresolvedFilterColumns(filters, new Set(['ServiceName', 'Body'])), + ).toEqual(['StatusCode']); + }); + + it('is empty when every attributable column resolves', () => { + expect( + unresolvedFilterColumns(filters, new Set(['ServiceName', 'StatusCode'])), + ).toEqual([]); + }); + + it('is empty (not excluding) while columns are still unknown', () => { + expect(unresolvedFilterColumns(filters, undefined)).toEqual([]); + }); +}); + +describe('resolveExtraColumnsForSource', () => { + it('projects the column where present and NULL where missing', () => { + expect( + resolveExtraColumnsForSource( + ['ServiceName', 'StatusCode', 'weird col'], + new Set(['ServiceName', 'weird col']), + ), + ).toEqual([ + { name: 'ServiceName', expression: 'ServiceName' }, + { name: 'StatusCode', expression: null }, + { name: 'weird col', expression: '`weird col`' }, + ]); + }); +}); diff --git a/packages/app/src/hooks/useMultiSourceSearch.ts b/packages/app/src/hooks/useMultiSourceSearch.ts new file mode 100644 index 0000000000..f4ecdb4da4 --- /dev/null +++ b/packages/app/src/hooks/useMultiSourceSearch.ts @@ -0,0 +1,148 @@ +import { useMemo } from 'react'; +import { + ColumnMeta, + filterColumnMetaByType, + JSDataType, +} from '@hyperdx/common-utils/dist/clickhouse'; +import { MultiSourceExtraColumn } from '@hyperdx/common-utils/dist/core/searchChartConfig'; +import { Filter, TSource } from '@hyperdx/common-utils/dist/types'; + +import { useColumns } from '@/hooks/useMetadata'; +import { useMultiSourceSlots } from '@/hooks/useSourceSlots'; + +const EMPTY_SOURCE_PARAMS = { + databaseName: '', + tableName: '', + connectionId: '', +}; + +function columnsParamsFor(source: TSource | undefined) { + if (source == null) return EMPTY_SOURCE_PARAMS; + return { + databaseName: source.from.databaseName, + tableName: source.from.tableName, + connectionId: source.connection, + }; +} + +export type MultiSourceColumnOption = { + name: string; + /** How many of the selected sources have this column. */ + availableCount: number; +}; + +/** Slot hook: DESCRIBE columns for one source. Stable — `.data` is cached. */ +function useSourceColumnsSlot( + source: TSource | undefined, +): ColumnMeta[] | undefined { + return useColumns(columnsParamsFor(source)).data; +} + +/** + * Top-level columns (DESCRIBE) for each selected source of a multi-source + * search, plus the deduped union with per-column availability counts for the + * add-column picker. useColumns self-disables for unused slots. + */ +export function useMultiSourceColumns(sources: TSource[]): { + columnsBySourceId: Map>; + unionColumns: MultiSourceColumnOption[]; + /** Union of Date/DateTime column name → ClickHouse type across sources. */ + dateTimeColumns: Map; +} { + const slotData = useMultiSourceSlots( + sources, + useSourceColumnsSlot, + undefined, + ); + + return useMemo(() => { + const columnsBySourceId = new Map>(); + const availability = new Map(); + const dateTimeColumns = new Map(); + + for (let i = 0; i < sources.length; i++) { + const source = sources[i]; + const columns = slotData[i]; + if (source == null || columns == null) continue; + const names = new Set(columns.map(c => c.name)); + columnsBySourceId.set(source.id, names); + for (const name of names) { + availability.set(name, (availability.get(name) ?? 0) + 1); + } + for (const col of filterColumnMetaByType(columns, [JSDataType.Date]) ?? + []) { + if (!dateTimeColumns.has(col.name)) { + dateTimeColumns.set(col.name, col.type); + } + } + } + + const unionColumns = [...availability.entries()] + .map(([name, availableCount]) => ({ name, availableCount })) + .sort( + (a, b) => + b.availableCount - a.availableCount || a.name.localeCompare(b.name), + ); + + return { columnsBySourceId, unionColumns, dateTimeColumns }; + }, [slotData, sources]); +} + +/** + * Root column a filter references, for per-source resolvability checks. + * sql_ast filters carry the escaped SQL key in `left` (e.g. `ServiceName`, + * a backticked identifier, or `LogAttributes['level']` whose root is + * `LogAttributes`). Other filter types (raw sql/lucene conditions) can't be + * attributed to a single column and return null — callers should apply them + * to every source and rely on per-source error isolation. + */ +export function filterRootColumn(filter: Filter): string | null { + if (filter.type !== 'sql_ast') return null; + const left = filter.left.trim(); + const backticked = left.match(/^`([^`]+)`/); + if (backticked) return backticked[1]; + const plain = left.match(/^[A-Za-z_][A-Za-z0-9_]*/); + return plain ? plain[0] : null; +} + +/** + * For one source: which of the active filters reference a column its table + * doesn't have. A non-empty result means the source can't answer the + * filtered search and should be excluded (with a visible reason). + */ +export function unresolvedFilterColumns( + filters: Filter[], + sourceColumns: Set | undefined, +): string[] { + if (sourceColumns == null) return []; + const missing = new Set(); + for (const filter of filters) { + const root = filterRootColumn(filter); + if (root != null && !sourceColumns.has(root)) { + missing.add(root); + } + } + return [...missing]; +} + +/** Quote a column name as a ClickHouse identifier when it needs it. */ +function quoteIdentifier(name: string): string { + return /^[A-Za-z_][A-Za-z0-9_]*$/.test(name) + ? name + : `\`${name.replace(/`/g, '\\`')}\``; +} + +/** + * Resolve the user-picked extra column names into per-source SELECT + * expressions: the (quoted) column itself where the source's table has it, + * NULL otherwise — so every source still returns the same result shape. + */ +export function resolveExtraColumnsForSource( + extraColumnNames: string[], + sourceColumns: Set | undefined, +): MultiSourceExtraColumn[] { + return extraColumnNames.map(name => ({ + name, + expression: sourceColumns?.has(name) ? quoteIdentifier(name) : null, + })); +} diff --git a/packages/app/src/hooks/useResolvedSourcesParam.ts b/packages/app/src/hooks/useResolvedSourcesParam.ts new file mode 100644 index 0000000000..deb2f449d1 --- /dev/null +++ b/packages/app/src/hooks/useResolvedSourcesParam.ts @@ -0,0 +1,58 @@ +import { useEffect, useMemo } from 'react'; +import { SourceKind, TSource } from '@hyperdx/common-utils/dist/types'; +import { notifications } from '@mantine/notifications'; + +import { MAX_SEARCH_SOURCES } from '@/defaults'; +import { useSources } from '@/source'; +import { resolveSourcesParam } from '@/utils/sourceParams'; + +const EMPTY_SOURCES: TSource[] = []; + +/** + * Resolves the multi-source search param (a list of source IDs or names) to + * the matching sources, deduped and capped at MAX_SEARCH_SOURCES. + * + * Elements that don't match any usable source are dropped from the selection + * and reported once via a Mantine warning, mirroring useResolvedSourceParam. + */ +export function useResolvedSourcesParam( + paramValues: string[] | null | undefined, + { kinds }: { kinds?: SourceKind[] } = {}, +): { sources: TSource[] } { + const { data: allSources } = useSources(); + + // Key the memo on a serialized `kinds` so callers can pass inline arrays + // without breaking memoization. + const kindsKey = kinds?.join(','); + const { sources, unresolvedKey } = useMemo(() => { + const allKinds = new Set(Object.values(SourceKind)); + const resolvedKinds = kindsKey + ? kindsKey.split(',').filter((k): k is SourceKind => allKinds.has(k)) + : undefined; + const resolution = resolveSourcesParam(paramValues, allSources, { + kinds: resolvedKinds, + max: MAX_SEARCH_SOURCES, + }); + if (resolution.status !== 'resolved') { + return { sources: EMPTY_SOURCES, unresolvedKey: undefined }; + } + return { + sources: resolution.sources.length ? resolution.sources : EMPTY_SOURCES, + unresolvedKey: resolution.unresolved.length + ? resolution.unresolved.join(', ') + : undefined, + }; + }, [paramValues, allSources, kindsKey]); + + useEffect(() => { + if (unresolvedKey == null) return; + notifications.show({ + id: 'sources-param-unresolved-' + unresolvedKey, + color: 'yellow', + title: 'Some sources were not found', + message: `No searchable source matches: ${unresolvedKey}. They may have been renamed or deleted.`, + }); + }, [unresolvedKey]); + + return useMemo(() => ({ sources }), [sources]); +} diff --git a/packages/app/src/hooks/useSourceSlots.ts b/packages/app/src/hooks/useSourceSlots.ts new file mode 100644 index 0000000000..04286e5832 --- /dev/null +++ b/packages/app/src/hooks/useSourceSlots.ts @@ -0,0 +1,32 @@ +import { useMemo } from 'react'; + +import { MAX_SEARCH_SOURCES } from '@/defaults'; + +/** + * Run one instance of a hook per selected source of a search. + * + * The rules of hooks require a constant hook count per component, but search + * needs one pipeline per selected source — and `useQueries` can't cover these + * pipelines (row streams are `useInfiniteQuery`-based, which has no plural + * form, and the chart/facet pipelines compose other hooks). So the hook count + * is pinned at MAX_SEARCH_SOURCES here, in one place: unused slots receive + * `undefined` and every slot hook is expected to self-disable for it. + * + * `useSlot` must be a stable, named hook (the rules-of-hooks lint understands + * `use*`-named parameters) and should return a memoized value, so the array + * this returns is referentially stable and safe to use in dependency lists. + * + * Lives in its own module so both the search hooks and the metadata hooks can + * use it without an import cycle. + */ +export function useMultiSourceSlots( + items: readonly Item[], + useSlot: (item: Item | undefined, opts: Opts) => Result, + opts: Opts, +): Result[] { + const s0 = useSlot(items[0], opts); + const s1 = useSlot(items[1], opts); + const s2 = useSlot(items[2], opts); + const count = Math.min(items.length, MAX_SEARCH_SOURCES); + return useMemo(() => [s0, s1, s2].slice(0, count), [s0, s1, s2, count]); +} From c6e26b259c3876221c88db2d0be9a5603fe60f49 Mon Sep 17 00:00:00 2001 From: Tom Alexander Date: Wed, 12 Aug 2026 13:07:32 -0400 Subject: [PATCH 6/8] feat(app): histogram and filters across every selected source MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The results table already spanned sources; the histogram and the filters sidebar still picked a component per case at the call site. The page now renders SearchHistogram and SearchTotalCount, which take the same per-source list the table does. One source keeps the full DBTimeChart — severity grouping, series drill-down and focus, the pinned tooltip — because a single source has a severity vocabulary to group by. Several sources can't share one, so they stack a count() series each. DBSearchPageFilters takes a list too, so the sidebar people already use works across a selection: facet fields and values merge, value counts are summed so a percentage describes the whole search, "load more" fans out and unions, and pins read as a union and write to every selected source. A source whose table lacks a filtered column is excluded from the results with the reason on its status chip, rather than quietly returning rows that ignore the filter. With one source every path is the one it was before: analysis-mode tabs, denoise, shared filters, and dropping percentages when a distribution query fails. --- .changeset/multi-source-filters.md | 12 + packages/app/src/DBSearchPage.tsx | 156 +++++-- .../DBSearchPage.directTrace.test.tsx | 4 + .../src/components/DBSearchPageFilters.tsx | 119 +++-- .../DBSearchPageFilters/NestedFilterGroup.tsx | 7 +- .../components/DBSearchPageFilters/hooks.ts | 145 ++++++ .../app/src/components/SearchHistogram.tsx | 421 ++++++++++++++++++ .../__tests__/DBSearchPageFilters.test.tsx | 59 +-- packages/app/src/hooks/useMetadata.tsx | 76 +++- packages/app/src/searchFilters.tsx | 100 +++++ 10 files changed, 1002 insertions(+), 97 deletions(-) create mode 100644 .changeset/multi-source-filters.md create mode 100644 packages/app/src/components/SearchHistogram.tsx diff --git a/.changeset/multi-source-filters.md b/.changeset/multi-source-filters.md new file mode 100644 index 0000000000..d933bb2629 --- /dev/null +++ b/.changeset/multi-source-filters.md @@ -0,0 +1,12 @@ +--- +'@hyperdx/app': minor +--- + +The filters sidebar works across every selected source. Facet fields and +values merge across sources, value counts are summed, "load more" fans out, +and pins (personal and team-shared) read as a union and apply to the whole +selection. Checking a value filters every source that has the field; a source +whose table lacks a filtered column is excluded from the results with a +visible reason on its status chip instead of silently returning unfiltered +rows. Filter pills and add-to-filter from the row side panel work across +sources too. diff --git a/packages/app/src/DBSearchPage.tsx b/packages/app/src/DBSearchPage.tsx index c11ff3efe1..caa99bf8df 100644 --- a/packages/app/src/DBSearchPage.tsx +++ b/packages/app/src/DBSearchPage.tsx @@ -32,6 +32,7 @@ import { } from '@hyperdx/common-utils/dist/clickhouse'; import { tcFromSource } from '@hyperdx/common-utils/dist/core/metadata'; import { + ALERT_COUNT_DEFAULT_SELECT, buildMultiSourceSearchConfig, buildSearchChartConfig, } from '@hyperdx/common-utils/dist/core/searchChartConfig'; @@ -92,7 +93,7 @@ import { AlertStatusIcon } from '@/components/AlertStatusIcon'; import { ContactSupportText } from '@/components/ContactSupportText'; import { DBSearchPageFilters } from '@/components/DBSearchPageFilters'; import { cleanClickHouseExpression } from '@/components/DBSearchPageFilters/utils'; -import { DBTimeChart, type SeriesGroupFilter } from '@/components/DBTimeChart'; +import { type SeriesGroupFilter } from '@/components/DBTimeChart'; import EmptyState from '@/components/EmptyState'; import { ErrorBoundary } from '@/components/Error/ErrorBoundary'; import { FavoriteButton } from '@/components/FavoriteButton'; @@ -100,12 +101,16 @@ import ResourceTerraformPopover from '@/components/Iac/ResourceTerraformPopover' import { InputControlled } from '@/components/InputControlled'; import MultiSourceColumnPicker from '@/components/MultiSourceColumnPicker'; import OnboardingModal from '@/components/OnboardingModal'; +import { + SearchHistogram, + type SearchHistogramSpec, + SearchTotalCount, +} from '@/components/SearchHistogram'; import SearchWhereInput, { getStoredLanguage, } from '@/components/SearchInput/SearchWhereInput'; import SearchPageActionBar from '@/components/SearchPageActionBar'; import SearchResultsTable from '@/components/SearchResultsTable'; -import SearchTotalCountChart from '@/components/SearchTotalCountChart'; import { SourceMultiSelectControlled } from '@/components/SourceMultiSelect'; import { TableSourceForm } from '@/components/Sources/SourceForm'; import { SourceSelectControlled } from '@/components/SourceSelect'; @@ -343,12 +348,12 @@ function ExpandFiltersButton({ onExpand }: { onExpand: () => void }) { function SearchResultsCountGroup({ isFilterSidebarCollapsed, onExpandFilters, - histogramTimeChartConfig, + histogramSpecs, enableParallelQueries, }: { isFilterSidebarCollapsed: boolean; onExpandFilters: () => void; - histogramTimeChartConfig: BuilderChartConfigWithDateRange; + histogramSpecs: SearchHistogramSpec[]; enableParallelQueries?: boolean; }) { return ( @@ -356,8 +361,8 @@ function SearchResultsCountGroup({ {isFilterSidebarCollapsed && ( )} - @@ -1785,6 +1790,45 @@ export function DBSearchPage() { searchedTimeRange, ]); + const multiHistogramSpecs = useMemo(() => { + if (!isMultiSource || isMultiSourceSqlBlocked) return []; + if ( + multiSourceFilters.length > 0 && + columnsBySourceId.size < searchedMultiSources.length + ) { + return []; + } + const where = searchedConfig.where ?? ''; + return searchedMultiSources.map(source => ({ + source, + disabledReason: multiDisabledReasons.get(source.id), + config: { + ...buildMultiSourceSearchConfig(source, { + where, + whereLanguage: 'lucene', + filters: multiSourceFilters, + }), + select: ALERT_COUNT_DEFAULT_SELECT, + orderBy: undefined, + granularity: 'auto' as const, + dateRange: searchedTimeRange, + displayType: DisplayType.StackedBar, + // Match the single-source histogram: reflect the user's exact range + // so chart and table counts agree (see histogramTimeChartConfig). + alignDateRangeToGranularity: false, + dateRangeEndInclusive: true, + }, + })); + }, [ + isMultiSource, + isMultiSourceSqlBlocked, + searchedMultiSources, + searchedConfig.where, + multiSourceFilters, + multiDisabledReasons, + columnsBySourceId, + searchedTimeRange, + ]); // --- End multi-source search --------------------------------------------- // query error handling @@ -2150,6 +2194,20 @@ export function DBSearchPage() { searchedConfig.select, ]); + // The chart/count query plan for however many sources are selected: one + // source keeps its severity-grouped histogram, several get one count() + // series each (stacked by source). + const histogramSpecs = useMemo(() => { + if (isMultiSource) return multiHistogramSpecs; + if (searchedSource == null || histogramTimeChartConfig == null) return []; + return [{ source: searchedSource, config: histogramTimeChartConfig }]; + }, [ + isMultiSource, + multiHistogramSpecs, + searchedSource, + histogramTimeChartConfig, + ]); + const onFormSubmit = useCallback>( e => { e.preventDefault(); @@ -2231,6 +2289,22 @@ export function DBSearchPage() { }; }, [chartConfig, searchedTimeRange, aliasWith]); + // The sidebar reads facets, values, and pins across everything selected; + // with one source that is exactly the single-source sidebar. + const filterSidebarSources = useMemo(() => { + if (isMultiSource) { + // Facet queries want each source's search shape (its own FROM, + // connection, and WHERE), not the aggregated histogram config. + return searchStreamSpecs.map(({ source, config }) => ({ + source, + config: { ...config, orderBy: undefined }, + })); + } + return searchedSource != null + ? [{ source: searchedSource, config: filtersChartConfig }] + : []; + }, [isMultiSource, searchStreamSpecs, searchedSource, filtersChartConfig]); + const openNewSourceModal = useCallback(() => { setNewSourceModalOpened(true); }, []); @@ -2789,7 +2863,7 @@ export function DBSearchPage() { height: '100%', }} > - {!isFilterSidebarCollapsed && !isMultiSource && ( + {!isFilterSidebarCollapsed && ( setIsFilterSidebarCollapsed(false) } - histogramTimeChartConfig={histogramTimeChartConfig} + histogramSpecs={histogramSpecs} /> - ) : ( <> - {/* The histogram, total count, and filters sidebar come - with the next change; searching several sources - returns the merged results table on its own. */} + + + + {isFilterSidebarCollapsed && ( + + setIsFilterSidebarCollapsed(false) + } + /> + )} + + + {shouldShowLiveModeHint && ( + + )} + + + + + setIsFilterSidebarCollapsed(false) } - histogramTimeChartConfig={histogramTimeChartConfig} + histogramSpecs={histogramSpecs} enableParallelQueries /> @@ -2990,14 +3095,9 @@ export function DBSearchPage() { className={searchPageStyles.timeChartContainer} mih="0" > - () =>
); // Multi-source components pull in DBRowSidePanel (and its deep import graph), // which this test isolates away just like DBSqlRowTableWithSidebar above. jest.mock('../components/SearchResultsTable', () => () =>
); +jest.mock('../components/SearchHistogram', () => ({ + SearchHistogram: () =>
, + SearchTotalCount: () =>
, +})); jest.mock('../components/PatternTable', () => () =>
); jest.mock('../components/Search/DBSearchHeatmapChart', () => ({ DBSearchHeatmapChart: () =>
, diff --git a/packages/app/src/components/DBSearchPageFilters.tsx b/packages/app/src/components/DBSearchPageFilters.tsx index 67a154aae7..55a60c41f7 100644 --- a/packages/app/src/components/DBSearchPageFilters.tsx +++ b/packages/app/src/components/DBSearchPageFilters.tsx @@ -8,6 +8,7 @@ import { FilterState } from '@hyperdx/common-utils/dist/filters'; import { BuilderChartConfigWithDateRange, SourceKind, + TSource, } from '@hyperdx/common-utils/dist/types'; import { Accordion, @@ -51,22 +52,23 @@ import { import { IS_CLICKHOUSE_BUILD } from '@/config'; import { useColumns, - useGetValuesDistribution, useJsonColumns, + useMergedValuesDistribution, useTableMetadata, } from '@/hooks/useMetadata'; +import { useMultiSourceColumns } from '@/hooks/useMultiSourceSearch'; import useResizable from '@/hooks/useResizable'; import { usePinnedFiltersApi } from '@/pinnedFilters'; import { FilterStateHook, IS_ROOT_SPAN_COLUMN_NAME, - usePinnedFilters, + usePinnedFiltersForSources, } from '@/searchFilters'; import { useSource } from '@/source'; import { useLocalStorage } from '@/utils'; import { FilterSettingsPanel } from './DBSearchPageFilters/FilterSettingsPopover'; -import { useFetchFacets } from './DBSearchPageFilters/hooks'; +import { useFetchFacetsForSources } from './DBSearchPageFilters/hooks'; import { NestedFilterGroup } from './DBSearchPageFilters/NestedFilterGroup'; import { PinShareIndicator, @@ -82,6 +84,17 @@ import { import resizeStyles from '@styles/ResizablePanel.module.scss'; import classes from '@styles/SearchPage.module.scss'; +// Placeholder used when no source is selected yet; nothing queries with it. +const EMPTY_FILTER_CHART_CONFIG = { + connection: '', + from: { databaseName: '', tableName: '' }, + timestampValueExpression: '', + select: '', + where: '', + whereLanguage: 'sql' as const, + dateRange: [new Date(0), new Date(0)] as [Date, Date], +}; + /* The initial number of values per filter to render */ const INITIAL_MAX_VALUES_DISPLAYED = 10; @@ -402,7 +415,8 @@ export type FilterGroupProps = { isDefaultExpanded?: boolean; showFilterCounts?: boolean; 'data-testid'?: string; - chartConfig: BuilderChartConfigWithDateRange; + /** One config per selected source; value counts are summed across them. */ + chartConfigs: BuilderChartConfigWithDateRange[]; isLive?: boolean; onRangeChange?: (range: { min: number; max: number }) => void; distributionKey?: string; @@ -428,7 +442,7 @@ const FilterGroupBody = ({ onLoadMore, loadMoreLoading, hasLoadedMore, - chartConfig, + chartConfigs, isLive, distributionKey, showDistributions, @@ -449,7 +463,7 @@ const FilterGroupBody = ({ onLoadMore: (key: string) => void; loadMoreLoading: boolean; hasLoadedMore: boolean; - chartConfig: BuilderChartConfigWithDateRange; + chartConfigs: BuilderChartConfigWithDateRange[]; isLive?: boolean; distributionKey?: string; showDistributions: boolean; @@ -463,16 +477,19 @@ const FilterGroupBody = ({ const [recentlyMoved, setRecentlyMoved] = useState>( new Set(), ); - // For live searches, don't refresh percentages when date range changes + // For live searches, don't refresh percentages when date range changes. + // Every source's config carries the same searched range, so the first one + // speaks for all of them. + const primaryDateRange = chartConfigs[0]?.dateRange; const [dateRange, setDateRange] = useState<[Date, Date]>( - chartConfig.dateRange, + primaryDateRange ?? EMPTY_FILTER_CHART_CONFIG.dateRange, ); useEffect(() => { - if (!isLive) { - setDateRange(chartConfig.dateRange); + if (!isLive && primaryDateRange != null) { + setDateRange(primaryDateRange); } - }, [chartConfig.dateRange, isLive]); + }, [primaryDateRange, isLive]); const handleSetSearch = useCallback( (value: string) => { @@ -484,25 +501,23 @@ const FilterGroupBody = ({ [hasLoadedMore, name, onLoadMore], ); + const distributionConfigs = useMemo( + () => chartConfigs.map(config => ({ ...config, dateRange })), + [chartConfigs, dateRange], + ); const { data: distributionData, isFetching: isFetchingDistribution, error: distributionError, - } = useGetValuesDistribution( + } = useMergedValuesDistribution( { - chartConfig: { ...chartConfig, dateRange }, + chartConfigs: distributionConfigs, key: distributionKey || name, limit: 100, // The 100 most common values are enough to find any values that are present in at least 1% of rows }, - { - enabled: showDistributions, - }, + { enabled: showDistributions }, ); - useEffect(() => { - onFetchingDistributionChange(isFetchingDistribution); - }, [isFetchingDistribution, onFetchingDistributionChange]); - useEffect(() => { if (distributionError) { notifications.show({ @@ -515,6 +530,10 @@ const FilterGroupBody = ({ } }, [distributionError, onDistributionError]); + useEffect(() => { + onFetchingDistributionChange(isFetchingDistribution); + }, [isFetchingDistribution, onFetchingDistributionChange]); + const totalAppliedFiltersSize = selectedValues.included.size + selectedValues.excluded.size + @@ -900,7 +919,7 @@ export const FilterGroup = ({ isDefaultExpanded, showFilterCounts, 'data-testid': dataTestId, - chartConfig, + chartConfigs, isLive, distributionKey, onRangeChange, @@ -1040,7 +1059,7 @@ export const FilterGroup = ({ onLoadMore={onLoadMore} loadMoreLoading={loadMoreLoading} hasLoadedMore={hasLoadedMore} - chartConfig={chartConfig} + chartConfigs={chartConfigs} isLive={isLive} distributionKey={distributionKey} showDistributions={showDistributions} @@ -1061,10 +1080,9 @@ const DBSearchPageFiltersComponent = ({ clearFilter, setFilterValue: _setFilterValue, isLive, - chartConfig, + sources, analysisMode, setAnalysisMode, - sourceId, showDelta, denoiseResults, setDenoiseResults, @@ -1076,8 +1094,11 @@ const DBSearchPageFiltersComponent = ({ analysisMode: 'results' | 'delta' | 'pattern'; setAnalysisMode: (mode: 'results' | 'delta' | 'pattern') => void; isLive: boolean; - chartConfig: BuilderChartConfigWithDateRange; - sourceId?: string; + /** + * One entry per selected source. Facets, values, and pins merge across all + * of them; a single source behaves exactly as before. + */ + sources: { source: TSource; config: BuilderChartConfigWithDateRange }[]; showDelta: boolean; denoiseResults: boolean; setDenoiseResults: (denoiseResults: boolean) => void; @@ -1096,6 +1117,23 @@ const DBSearchPageFiltersComponent = ({ }, [_setFilterValue], ); + // The first selected source is the "primary": it anchors the things that + // are inherently about one table (schema preview, the analysis-mode tabs). + // Everything users read or click in the list itself merges across sources. + const primarySource = sources[0]?.source; + const sourceId = primarySource?.id; + const chartConfig = sources[0]?.config ?? EMPTY_FILTER_CHART_CONFIG; + const sourceIds = useMemo(() => sources.map(s => s.source.id), [sources]); + const chartConfigs = useMemo(() => sources.map(s => s.config), [sources]); + const facetSpecs = useMemo( + () => + sources.map(({ source, config }) => ({ + sourceId: source.id, + chartConfig: config, + })), + [sources], + ); + const { toggleFilterPin, toggleFieldPin, @@ -1111,7 +1149,7 @@ const DBSearchPageFiltersComponent = ({ resetSharedFilters, hasPersonalPins, hasSharedPins, - } = usePinnedFilters(sourceId ?? null); + } = usePinnedFiltersForSources(sourceIds); const { data: pinnedFiltersApiData } = usePinnedFiltersApi(sourceId ?? null); const [isSharedFiltersVisible, setSharedFiltersVisible] = useLocalStorage( 'hdx-shared-filters-visible', @@ -1144,6 +1182,11 @@ const DBSearchPageFiltersComponent = ({ chartConfig.dateRange, ); + // Filter keys can come from any selected source's schema, so escaping is + // resolved against the union of their columns. + const { columnsBySourceId } = useMultiSourceColumns( + useMemo(() => sources.map(s => s.source), [sources]), + ); const { data: columns } = useColumns({ databaseName: chartConfig.from.databaseName, tableName: chartConfig.from.tableName, @@ -1164,10 +1207,13 @@ const DBSearchPageFiltersComponent = ({ // Conditionally backtick-quote facet keys that contain special characters and // match known column names, so they can be used in the ClickHouse query to get // key values. - const knownColumns = useMemo( - () => (columns ? new Set(columns.map(c => c.name)) : new Set()), - [columns], - ); + const knownColumns = useMemo(() => { + const names = new Set(columns?.map(c => c.name) ?? []); + for (const sourceColumns of columnsBySourceId.values()) { + for (const name of sourceColumns) names.add(name); + } + return names; + }, [columns, columnsBySourceId]); const [showMoreFields, setShowMoreFields] = useState(false); const { @@ -1178,9 +1224,8 @@ const DBSearchPageFiltersComponent = ({ loadMoreFacetsForKey, loadMoreLoadingKeys, extraFacetKeys, - } = useFetchFacets({ - chartConfig, - sourceId: sourceId ?? null, + } = useFetchFacetsForSources({ + specs: facetSpecs, dateRange, mode: showAllValues ? 'all' : 'exact', filterState, @@ -1516,7 +1561,7 @@ const DBSearchPageFiltersComponent = ({ ); }) } - chartConfig={chartConfig} + chartConfigs={chartConfigs} isLive={isLive} /> ))} @@ -1571,7 +1616,7 @@ const DBSearchPageFiltersComponent = ({ entry.range != null))) ); })()} - chartConfig={chartConfig} + chartConfigs={chartConfigs} isLive={isLive} onRangeChange={range => setFilterRange(facet.key, range)} /> @@ -1598,7 +1643,7 @@ const DBSearchPageFiltersComponent = ({ loadMoreLoadingKeys, showFilterCounts, isFacetsLoading, - chartConfig, + chartConfigs, isLive, setFilterRange, tableMetadata, diff --git a/packages/app/src/components/DBSearchPageFilters/NestedFilterGroup.tsx b/packages/app/src/components/DBSearchPageFilters/NestedFilterGroup.tsx index 4145d8e917..6291a6136c 100644 --- a/packages/app/src/components/DBSearchPageFilters/NestedFilterGroup.tsx +++ b/packages/app/src/components/DBSearchPageFilters/NestedFilterGroup.tsx @@ -39,7 +39,8 @@ type NestedFilterGroupProps = { hasLoadedMore: Record; isDefaultExpanded?: boolean; 'data-testid'?: string; - chartConfig: any; // Using any to avoid importing ChartConfigWithDateRange + /** One config per selected source; counts are summed across them. */ + chartConfigs: any[]; // `any` avoids importing ChartConfigWithDateRange isLive?: boolean; }; @@ -70,7 +71,7 @@ export const NestedFilterGroup = ({ hasLoadedMore, isDefaultExpanded, 'data-testid': dataTestId, - chartConfig, + chartConfigs, isLive, }: NestedFilterGroupProps) => { const selectedValues: FilterState = useMemo( @@ -253,7 +254,7 @@ export const NestedFilterGroup = ({ hasLoadedMore={hasLoadedMore[child.key] || false} showFilterCounts={showFilterCounts} isDefaultExpanded={childHasSelections} - chartConfig={chartConfig} + chartConfigs={chartConfigs} isLive={isLive} />
diff --git a/packages/app/src/components/DBSearchPageFilters/hooks.ts b/packages/app/src/components/DBSearchPageFilters/hooks.ts index a9d676102a..d9dd4319e6 100644 --- a/packages/app/src/components/DBSearchPageFilters/hooks.ts +++ b/packages/app/src/components/DBSearchPageFilters/hooks.ts @@ -17,6 +17,7 @@ import { useMapColumns, useMetadataWithSettings, } from '@/hooks/useMetadata'; +import { useMultiSourceSlots } from '@/hooks/useSourceSlots'; import { escapeFilterStateKeys, usePinnedFilters } from '@/searchFilters'; import { useSource } from '@/source'; import { mergePath } from '@/utils'; @@ -372,3 +373,147 @@ export function useFetchFacets({ extraFacetKeys, }; } + +export type SourceFacetSpec = { + sourceId: string; + chartConfig: BuilderChartConfigWithDateRange; +}; + +/** Slot hook: the full facet pipeline for one selected source. */ +function useSourceFacetsSlot( + spec: SourceFacetSpec | undefined, + opts: { + dateRange: [Date, Date]; + mode: 'all' | 'exact'; + filterState?: FilterState; + showMoreFields?: boolean; + }, +) { + const query = useFetchFacets({ + chartConfig: spec?.chartConfig ?? STUB_FACET_CONFIG, + sourceId: spec?.sourceId ?? null, + dateRange: opts.dateRange, + mode: opts.mode, + filterState: opts.filterState, + showMoreFields: opts.showMoreFields, + enabled: spec != null, + }); + return query; +} + +const STUB_FACET_CONFIG: BuilderChartConfigWithDateRange = { + connection: '', + from: { databaseName: '', tableName: '' }, + timestampValueExpression: '', + select: '', + where: '', + whereLanguage: 'sql', + dateRange: [new Date(0), new Date(0)], +}; + +/** + * Facets across every selected source: fields and values merged by field + * path, values unioned in first-seen order. "Load more" fans out to each + * source and unions what comes back, so a high-cardinality field expands + * across the whole search rather than one table. + * + * With a single source this is `useFetchFacets` for that source, unchanged. + */ +export function useFetchFacetsForSources({ + specs, + dateRange, + mode, + filterState, + showMoreFields, +}: { + specs: SourceFacetSpec[]; + dateRange: [Date, Date]; + mode: 'all' | 'exact'; + filterState?: FilterState; + showMoreFields?: boolean; +}) { + const slots = useMultiSourceSlots(specs, useSourceFacetsSlot, { + dateRange, + mode, + filterState, + showMoreFields, + }); + + const merged = useMemo(() => { + const byKey = new Map< + string, + { values: (string | boolean)[]; seen: Set } + >(); + let sawAny = false; + for (const slot of slots) { + const facets = slot.data.keyValues; + if (facets == null) continue; + sawAny = true; + for (const facet of facets) { + let entry = byKey.get(facet.key); + if (entry == null) { + entry = { values: [], seen: new Set() }; + byKey.set(facet.key, entry); + } + for (const value of facet.value) { + if (!entry.seen.has(value)) { + entry.seen.add(value); + entry.values.push(value); + } + } + } + } + const keyValues = sawAny + ? [...byKey.entries()].map(([key, entry]) => ({ + key, + value: entry.values, + })) + : undefined; + + const keys = slots.flatMap(slot => slot.data.keys ?? []); + const seenPaths = new Set(); + const mergedKeys = keys.filter(field => { + const id = `${field.path.join('.')}|${field.type}`; + if (seenPaths.has(id)) return false; + seenPaths.add(id); + return true; + }); + + return { keys: mergedKeys.length > 0 ? mergedKeys : undefined, keyValues }; + }, [slots]); + + const loadMoreFacetsForKey = useCallback( + async (key: string) => { + await Promise.all(slots.map(slot => slot.loadMoreFacetsForKey(key))); + }, + [slots], + ); + + const loadMoreLoadingKeys = useMemo(() => { + const keys = new Set(); + for (const slot of slots) { + for (const key of slot.loadMoreLoadingKeys) keys.add(key); + } + return keys; + }, [slots]); + + const extraFacetKeys = useMemo(() => { + const keys = new Set(); + for (const slot of slots) { + for (const key of slot.extraFacetKeys) keys.add(key); + } + return keys; + }, [slots]); + + return { + data: merged, + isLoading: slots.some(s => s.isLoading), + isFetching: slots.some(s => s.isFetching), + // A single failing source shouldn't blank the sidebar; surface the first. + error: slots.find(s => s.error != null)?.error, + loadMoreFacetsForKey, + loadMoreLoadingKeys, + extraFacetKeys, + areExtraFacetsLoading: slots.some(s => s.areExtraFacetsLoading), + }; +} diff --git a/packages/app/src/components/SearchHistogram.tsx b/packages/app/src/components/SearchHistogram.tsx new file mode 100644 index 0000000000..55dbaf1481 --- /dev/null +++ b/packages/app/src/components/SearchHistogram.tsx @@ -0,0 +1,421 @@ +import { useMemo, useState } from 'react'; +import { + ColumnMetaType, + filterColumnMetaByType, + JSDataType, + ResponseJSON, +} from '@hyperdx/common-utils/dist/clickhouse'; +import { + BuilderChartConfigWithDateRange, + DisplayType, + TSource, +} from '@hyperdx/common-utils/dist/types'; +import { Text } from '@mantine/core'; +import { keepPreviousData } from '@tanstack/react-query'; + +import api from '@/api'; +import { + convertToTimeChartConfig, + formatResponseForTimeChart, + useTimeChartSettings, +} from '@/ChartUtils'; +import ChartContainer from '@/components/charts/ChartContainer'; +import ChartErrorState from '@/components/charts/ChartErrorState'; +import { type ActiveClickPayload, MemoChart } from '@/HDXMultiSeriesTimeChart'; +import { useQueriedChartConfig } from '@/hooks/useChartConfig'; +import { useMultiSourceSlots } from '@/hooks/useSourceSlots'; +import type { NumberFormat } from '@/types'; + +import { DBTimeChart, type SeriesGroupFilter } from './DBTimeChart'; +import { getMultiSourceColor } from './MultiSourceBadge'; +import SearchTotalCountChart from './SearchTotalCountChart'; + +/** Synthetic group column tagged onto each source's histogram rows. */ +const SOURCE_GROUP_COLUMN = '__hdx_source'; + +export type SearchHistogramSpec = { + source: TSource; + /** Per-source count() histogram config (canonical WHERE, no groupBy). */ + config: BuilderChartConfigWithDateRange; + /** When set, the source doesn't run (mirrors MultiSourceStreamSpec). */ + disabledReason?: string; +}; + +// Placeholder for unused hook slots; never queried (enabled: false). +const STUB_CONFIG: BuilderChartConfigWithDateRange = { + connection: '', + from: { databaseName: '', tableName: '' }, + timestampValueExpression: '', + select: '', + where: '', + whereLanguage: 'sql', + dateRange: [new Date(0), new Date(0)], +}; + +type HistogramSlotState = { + data: ReturnType['data']; + isLoading: boolean; + isError: boolean; + error: Error | null; +}; + +function useHistogramSlot( + spec: SearchHistogramSpec | undefined, + { + enabled, + queryKeyPrefix, + enableParallelQueries, + parallelizeWhenPossible, + }: { + enabled: boolean; + queryKeyPrefix: string; + enableParallelQueries?: boolean; + parallelizeWhenPossible?: boolean; + }, +): HistogramSlotState { + const queriedConfig = useMemo( + () => convertToTimeChartConfig(spec?.config ?? STUB_CONFIG), + [spec?.config], + ); + + const { data, isLoading, isError, error } = useQueriedChartConfig( + queriedConfig, + { + // Key shape mirrors DBTimeChart/SearchTotalCountChart so TanStack can + // de-dupe the histogram and total-count consumers of the same source. + queryKey: [ + queryKeyPrefix, + queriedConfig, + 'chunked', + { + disableQueryChunking: false, + enableParallelQueries, + parallelizeWhenPossible, + }, + ], + placeholderData: keepPreviousData, + enableQueryChunking: true, + enableParallelQueries: enableParallelQueries && parallelizeWhenPossible, + enabled: enabled && spec != null && spec.disabledReason == null, + }, + ); + + // Stable identity per content change so the slots array (and everything + // memoized on it) doesn't churn on unrelated renders. + return useMemo( + () => ({ data, isLoading, isError, error: error ?? null }), + [data, isLoading, isError, error], + ); +} + +/** + * Runs one count() histogram query per selected source (one hook slot per + * source, see useMultiSourceSlots) and merges the responses into a single + * response shape with a synthetic source-name group column — so the standard + * time-chart transform naturally yields one series per source. + */ +function useMultiSourceHistogram( + specs: SearchHistogramSpec[], + { + enabled = true, + queryKeyPrefix, + enableParallelQueries, + }: { + enabled?: boolean; + queryKeyPrefix: string; + enableParallelQueries?: boolean; + }, +) { + const { data: me, isLoading: isLoadingMe } = api.useMe(); + const slots = useMultiSourceSlots(specs, useHistogramSlot, { + enabled: enabled && !isLoadingMe, + queryKeyPrefix, + enableParallelQueries, + parallelizeWhenPossible: me?.team?.parallelizeWhenPossible, + }); + + const isLoading = slots.some(s => s.isLoading); + const allFailed = slots.length > 0 && slots.every(s => s.isError); + const anyError = slots.some(s => s.isError); + const error = slots.find(s => s.error != null)?.error ?? undefined; + const isComplete = + slots.length > 0 && slots.every(s => s.isError || !!s.data?.isComplete); + + const mergedResponse: ResponseJSON> | undefined = + useMemo(() => { + let meta: ColumnMetaType[] | undefined; + const data: Record[] = []; + for (let i = 0; i < specs.length; i++) { + const response = slots[i]?.data; + if (response?.meta == null || response.meta.length === 0) continue; + if (meta == null) { + meta = [ + ...response.meta, + { name: SOURCE_GROUP_COLUMN, type: 'String' }, + ]; + } + const sourceName = specs[i].source.name; + for (const row of response.data ?? []) { + data.push({ ...row, [SOURCE_GROUP_COLUMN]: sourceName }); + } + } + return meta ? { data, meta, rows: data.length } : undefined; + }, [slots, specs]); + + return { mergedResponse, isLoading, allFailed, anyError, error, isComplete }; +} + +const EMPTY_NUMBER_FORMATS = new Map(); + +/** + * The multi-source search histogram: one stacked count() series per selected + * source, colored consistently with the results-table badges. A thin + * counterpart to DBTimeChart — drag-to-zoom and the legend work; per-series + * drill-down/pinned tooltips are single-source features and are omitted. + */ +function MergedSourcesTimeChart({ + specs, + enabled = true, + queryKeyPrefix, + enableParallelQueries, + onTimeRangeSelect, + showLegend = true, +}: { + specs: SearchHistogramSpec[]; + enabled?: boolean; + queryKeyPrefix: string; + enableParallelQueries?: boolean; + onTimeRangeSelect?: (start: Date, end: Date) => void; + showLegend?: boolean; +}) { + const { mergedResponse, isLoading, allFailed, error, isComplete } = + useMultiSourceHistogram(specs, { + enabled, + queryKeyPrefix, + enableParallelQueries, + }); + + const firstConfig = specs[0]?.config; + const { dateRange, granularity } = useTimeChartSettings( + firstConfig ?? STUB_CONFIG, + ); + + const [activeClickPayload, setActiveClickPayload] = useState< + ActiveClickPayload | undefined + >(); + + const colorBySourceName = useMemo( + () => + new Map( + specs.map((spec, i) => [spec.source.name, getMultiSourceColor(i)]), + ), + [specs], + ); + + const formatted = useMemo(() => { + if (mergedResponse == null) { + return null; + } + try { + const result = formatResponseForTimeChart({ + currentPeriodResponse: mergedResponse, + dateRange, + granularity, + generateEmptyBuckets: true, + }); + // One series per source: recolor to the shared per-source palette so + // the histogram matches the table badges and status chips. + for (const line of result.lineData) { + const color = colorBySourceName.get(line.dataKey); + if (color != null) { + line.color = color; + } + } + return result; + } catch (e) { + console.error(e); + return null; + } + }, [mergedResponse, dateRange, granularity, colorBySourceName]); + + if (allFailed && error) { + return ; + } + + return ( + + {isLoading && formatted == null ? ( +
+ Loading Chart Data... +
+ ) : formatted == null || formatted.graphResults.length === 0 ? ( +
+ No data found within time range. +
+ ) : ( + + )} + + ); +} + +/** + * Summed "N Results" across every selected source, sharing the histogram's + * per-source queries (identical query keys) so it adds no ClickHouse load. + */ +function MergedSourcesTotalCount({ + specs, + enabled = true, + queryKeyPrefix, + enableParallelQueries, +}: { + specs: SearchHistogramSpec[]; + enabled?: boolean; + queryKeyPrefix: string; + enableParallelQueries?: boolean; +}) { + const { mergedResponse, isLoading, allFailed } = useMultiSourceHistogram( + specs, + { + enabled, + queryKeyPrefix, + enableParallelQueries, + }, + ); + + const totalCount = useMemo(() => { + if (mergedResponse == null) return undefined; + // The count column may be renamed (e.g. via materialized views); fall back + // to the first numeric column, mirroring SearchTotalCountChart. + const countColumn = + mergedResponse.meta?.find(c => c.name === 'count()')?.name ?? + filterColumnMetaByType(mergedResponse.meta ?? [], [ + JSDataType.Number, + ])?.[0]?.name ?? + 'count()'; + return mergedResponse.data.reduce( + (sum: number, row: any) => sum + (Number.parseInt(row[countColumn]) || 0), + 0, + ); + }, [mergedResponse]); + + return ( + + {isLoading && totalCount == null ? ( + ··· Results + ) : totalCount != null && !allFailed ? ( + `${totalCount.toLocaleString()} Results` + ) : ( + '0 Results' + )} + + ); +} + +/** + * The search page's histogram, for any number of selected sources. + * + * One source keeps the full-featured DBTimeChart — series drill-down, focus, + * the pinned tooltip, MV optimization — grouped by severity/status, which is + * what a single source's chart has always shown. Several sources can't share + * a severity vocabulary, so they stack one count() series per source instead, + * and the merged chart trades the per-series drill-down for that. + */ +export function SearchHistogram({ + specs, + enabled = true, + queryKeyPrefix, + enableParallelQueries, + onTimeRangeSelect, + onFocusSeries, + showLegend, +}: { + /** One spec per selected source; N=1 is the single-source histogram. */ + specs: SearchHistogramSpec[]; + enabled?: boolean; + queryKeyPrefix: string; + enableParallelQueries?: boolean; + onTimeRangeSelect?: (start: Date, end: Date) => void; + /** Focus a severity/status series into the search (single source only). */ + onFocusSeries?: (filters: SeriesGroupFilter[]) => void; + showLegend?: boolean; +}) { + if (specs.length === 1) { + return ( + + ); + } + + return ( + + ); +} + +/** + * "N Results" for any number of sources: the single-source count query, or the + * sum across sources. Shares the histogram's per-source queries either way, so + * it adds no ClickHouse load. + */ +export function SearchTotalCount({ + specs, + enabled = true, + queryKeyPrefix, + enableParallelQueries, +}: { + specs: SearchHistogramSpec[]; + enabled?: boolean; + queryKeyPrefix: string; + enableParallelQueries?: boolean; +}) { + if (specs.length === 1) { + return ( + + ); + } + + return ( + + ); +} diff --git a/packages/app/src/components/__tests__/DBSearchPageFilters.test.tsx b/packages/app/src/components/__tests__/DBSearchPageFilters.test.tsx index 0353c68d4f..a1d4359c93 100644 --- a/packages/app/src/components/__tests__/DBSearchPageFilters.test.tsx +++ b/packages/app/src/components/__tests__/DBSearchPageFilters.test.tsx @@ -1,4 +1,3 @@ -import { UseQueryResult } from '@tanstack/react-query'; import { screen, within } from '@testing-library/react'; import userEvent from '@testing-library/user-event'; @@ -13,7 +12,7 @@ import { groupFacetsByBaseName, parseMapFieldName, } from '@/components/DBSearchPageFilters/utils'; -import { useGetValuesDistribution } from '@/hooks/useMetadata'; +import { useMergedValuesDistribution } from '@/hooks/useMetadata'; describe('cleanClickHouseExpression', () => { it('should remove toString wrapper', () => { @@ -50,9 +49,9 @@ describe('cleanClickHouseExpression', () => { }); jest.mock('@/hooks/useMetadata', () => ({ - useGetValuesDistribution: jest + useMergedValuesDistribution: jest .fn() - .mockReturnValue({ data: undefined, isFetching: false, error: undefined }), + .mockReturnValue({ data: undefined, isFetching: false, error: null }), })); describe('cleanedFacetName', () => { @@ -396,18 +395,20 @@ describe('FilterGroup', () => { loadMoreLoading: false, hasLoadedMore: false, isDefaultExpanded: true, - chartConfig: { - from: { - databaseName: 'test_db', - tableName: 'test_table', + chartConfigs: [ + { + from: { + databaseName: 'test_db', + tableName: 'test_table', + }, + select: '', + where: '', + whereLanguage: 'sql', + timestampValueExpression: '', + connection: 'test_connection', + dateRange: [new Date('2024-01-01'), new Date('2024-01-02')], }, - select: '', - where: '', - whereLanguage: 'sql', - timestampValueExpression: '', - connection: 'test_connection', - dateRange: [new Date('2024-01-01'), new Date('2024-01-02')], - }, + ], }; it('should sort options alphabetically by default', () => { @@ -441,7 +442,7 @@ describe('FilterGroup', () => { }); it('should show selected items first, then sort by counts, if percentages when they are enabled', () => { - jest.mocked(useGetValuesDistribution).mockReturnValue({ + jest.mocked(useMergedValuesDistribution).mockReturnValue({ data: new Map([ ['apple', 30], ['banana', 20], @@ -449,7 +450,7 @@ describe('FilterGroup', () => { ]), isFetching: false, error: null, - } as UseQueryResult>); + }); renderWithMantine( { }); it('should show percentages, if enabled', async () => { - jest.mocked(useGetValuesDistribution).mockReturnValue({ + jest.mocked(useMergedValuesDistribution).mockReturnValue({ data: new Map([ ['apple', 99.2], ['zebra', 0.6], ]), isFetching: false, error: null, - } as UseQueryResult>); + }); renderWithMantine( { onLoadMore: jest.fn(), loadMoreLoading: {} as Record, hasLoadedMore: {} as Record, - chartConfig: { - from: { databaseName: 'test_db', tableName: 'test_table' }, - select: '', - where: '', - whereLanguage: 'sql', - timestampValueExpression: '', - connection: 'test_connection', - dateRange: [new Date('2024-01-01'), new Date('2024-01-02')], - }, + chartConfigs: [ + { + from: { databaseName: 'test_db', tableName: 'test_table' }, + select: '', + where: '', + whereLanguage: 'sql', + timestampValueExpression: '', + connection: 'test_connection', + dateRange: [new Date('2024-01-01'), new Date('2024-01-02')], + }, + ], }; it('should not render child FilterGroups when collapsed', () => { diff --git a/packages/app/src/hooks/useMetadata.tsx b/packages/app/src/hooks/useMetadata.tsx index fca1559d5d..7e9acfda89 100644 --- a/packages/app/src/hooks/useMetadata.tsx +++ b/packages/app/src/hooks/useMetadata.tsx @@ -32,6 +32,7 @@ import api from '@/api'; import { IS_LOCAL_MODE } from '@/config'; import { LOCAL_STORE_CONNECTIONS_KEY } from '@/connection'; import { DEFAULT_FILTER_KEYS_FETCH_LIMIT } from '@/defaults'; +import { useMultiSourceSlots } from '@/hooks/useSourceSlots'; import { getMetadata } from '@/metadata'; import { useSource, useSources } from '@/source'; import { toArray } from '@/utils'; @@ -449,7 +450,7 @@ export function useMultipleGetKeyValues( }; } -export function useGetValuesDistribution( +function useGetValuesDistribution( { chartConfig, key, @@ -484,6 +485,79 @@ export function useGetValuesDistribution( }); } +/** Slot hook: value→count distribution for one source's config. */ +function useValuesDistributionSlot( + chartConfig: BuilderChartConfigWithDateRange | undefined, + { key, limit, enabled }: { key: string; limit: number; enabled: boolean }, +) { + const { data, isFetching, error } = useGetValuesDistribution( + { + chartConfig: chartConfig ?? STUB_DISTRIBUTION_CONFIG, + key, + limit, + }, + { enabled: enabled && chartConfig != null }, + ); + return useMemo( + () => ({ data, isFetching, error }), + [data, isFetching, error], + ); +} + +const STUB_DISTRIBUTION_CONFIG: BuilderChartConfigWithDateRange = { + connection: '', + from: { databaseName: '', tableName: '' }, + timestampValueExpression: '', + select: '', + where: '', + whereLanguage: 'sql', + dateRange: [new Date(0), new Date(0)], +}; + +/** + * Value→count distribution for a filter key across every selected source, + * summed. With one source this is exactly `useGetValuesDistribution`; with + * several, a value's count is the total across the sources that have it, so + * the sidebar's percentages describe the whole search rather than one table. + */ +export function useMergedValuesDistribution( + { + chartConfigs, + key, + limit, + }: { + chartConfigs: BuilderChartConfigWithDateRange[]; + key: string; + limit: number; + }, + options?: { enabled?: boolean }, +) { + const slots = useMultiSourceSlots(chartConfigs, useValuesDistributionSlot, { + key, + limit, + enabled: options?.enabled ?? true, + }); + + return useMemo(() => { + const merged = new Map(); + let any = false; + for (const slot of slots) { + if (slot.data == null) continue; + any = true; + for (const [value, count] of slot.data) { + merged.set(value, (merged.get(value) ?? 0) + count); + } + } + return { + data: any ? merged : undefined, + isFetching: slots.some(s => s.isFetching), + // Surface the first failure so the group can drop percentages rather + // than showing counts that silently omit a source. + error: slots.find(s => s.error != null)?.error ?? null, + }; + }, [slots]); +} + export function useGetKeyValues( { chartConfig, diff --git a/packages/app/src/searchFilters.tsx b/packages/app/src/searchFilters.tsx index ac193b7f26..5492f16a71 100644 --- a/packages/app/src/searchFilters.tsx +++ b/packages/app/src/searchFilters.tsx @@ -11,6 +11,7 @@ import { cleanClickHouseExpression, toQuotedClickHouseKeyExpression, } from './components/DBSearchPageFilters/utils'; +import { useMultiSourceSlots } from './hooks/useSourceSlots'; import { usePinnedFiltersApi, useUpdatePinnedFilters } from './pinnedFilters'; import { useLocalStorage } from './utils'; @@ -658,3 +659,102 @@ export function usePinnedFilters(sourceId: string | null) { hasSharedPins, }; } + +/** + * Pinned filters across every selected source. + * + * Pins are stored per source (personal pins in localStorage, shared pins in + * Mongo), so a search spanning several sources reads the union of their pins + * and writes to all of them: pinning `ServiceName` while searching logs and + * traces keeps it pinned in both, and unpinning clears it from both. With one + * source this is exactly `usePinnedFilters` for that source. + */ +export function usePinnedFiltersForSources(sourceIds: string[]) { + const slots = useMultiSourceSlots(sourceIds, usePinnedFiltersSlot, undefined); + const active = useMemo( + () => slots.slice(0, sourceIds.length), + [slots, sourceIds.length], + ); + + return useMemo(() => { + // Reads: a pin on any selected source shows in the sidebar. + const anyOf = + ( + pick: (s: PinnedFiltersHook) => (...a: A) => boolean, + ) => + (...args: A) => + active.some(slot => pick(slot)(...args)); + // Writes: fan out so a pin applies to the whole selection. Toggling is + // resolved against the merged view first, so a mixed state (pinned on one + // source, not another) resolves to "pin everywhere" rather than flipping + // each source independently. + const fanOut = + ( + pick: (s: PinnedFiltersHook) => (...a: A) => void, + isSet: (s: PinnedFiltersHook, ...a: A) => boolean, + ) => + (...args: A) => { + const shouldPin = !active.some(slot => isSet(slot, ...args)); + for (const slot of active) { + if (isSet(slot, ...args) !== shouldPin) { + pick(slot)(...args); + } + } + }; + + return { + pinnedFilters: active.reduce( + (acc, slot) => mergePinnedFilterValues(acc, slot.pinnedFilters), + {}, + ), + getPinnedFields: () => [ + ...new Set(active.flatMap(slot => slot.getPinnedFields())), + ], + isFilterPinned: anyOf(s => s.isFilterPinned), + isFieldPinned: anyOf(s => s.isFieldPinned), + isSharedFilterPinned: anyOf(s => s.isSharedFilterPinned), + isSharedFieldPinned: anyOf(s => s.isSharedFieldPinned), + toggleFilterPin: fanOut( + s => s.toggleFilterPin, + (s, property: string, value: string | boolean) => + s.isFilterPinned(property, value), + ), + toggleFieldPin: fanOut( + s => s.toggleFieldPin, + (s, key: string) => s.isFieldPinned(key), + ), + toggleSharedFilterPin: fanOut( + s => s.toggleSharedFilterPin, + (s, property: string, value: string | boolean) => + s.isSharedFilterPinned(property, value), + ), + toggleSharedFieldPin: fanOut( + s => s.toggleSharedFieldPin, + (s, key: string) => s.isSharedFieldPinned(key), + ), + resetPersonalPins: () => active.forEach(s => s.resetPersonalPins()), + resetSharedFilters: () => active.forEach(s => s.resetSharedFilters()), + hasPersonalPins: active.some(s => s.hasPersonalPins), + hasSharedPins: active.some(s => s.hasSharedPins), + }; + }, [active]); +} + +type PinnedFiltersHook = ReturnType; + +/** Slot hook: one source's pins. Unused slots pass null and stay inert. */ +function usePinnedFiltersSlot(sourceId: string | undefined) { + return usePinnedFilters(sourceId ?? null); +} + +/** Union two pinned-filter maps, deduping values per key. */ +function mergePinnedFilterValues( + a: PinnedFilters, + b: PinnedFilters, +): PinnedFilters { + const out: PinnedFilters = { ...a }; + for (const [key, values] of Object.entries(b)) { + out[key] = [...new Set([...(out[key] ?? []), ...values])]; + } + return out; +} From de0df53a996c98d4c99aea8cc43032cac6640def Mon Sep 17 00:00:00 2001 From: Tom Alexander Date: Wed, 12 Aug 2026 16:25:48 -0400 Subject: [PATCH 7/8] feat(app): support SQL search across sources, and say when one is excluded MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Multi-source search refused SQL, on the grounds that a SQL WHERE names one table's columns. That reasoning doesn't hold: the hazard it describes — a column name that exists in two sources meaning different things — applies just as much to Lucene, which was allowed. And the case it was really guarding against, a column one source lacks, has a better answer than refusing the language. Both languages now resolve the columns a search references against each source's schema before querying. A source with the columns runs the query; a source without them is excluded and its chip says which column is missing. That also closes a quieter gap: Lucene resolves an unknown field to a condition matching no rows, so `StatusCode:Error` across logs and traces used to drop the log source with nothing on screen to say so. Extraction is deliberately conservative — a query that can't be parsed, or a reference that isn't rooted at a plain column, excludes nobody and falls back to running the query and surfacing any real error per source. --- .changeset/multi-source-sql-and-exclusion.md | 12 ++ packages/app/src/DBSearchPage.tsx | 187 +++++++++--------- .../__tests__/useMultiSourceSearch.test.ts | 55 ++++++ .../app/src/hooks/useMultiSourceSearch.ts | 59 ++++-- .../src/__tests__/queryParser.test.ts | 58 ++++++ packages/common-utils/src/queryParser.ts | 47 +++++ 6 files changed, 309 insertions(+), 109 deletions(-) create mode 100644 .changeset/multi-source-sql-and-exclusion.md diff --git a/.changeset/multi-source-sql-and-exclusion.md b/.changeset/multi-source-sql-and-exclusion.md new file mode 100644 index 0000000000..96a8ec14a3 --- /dev/null +++ b/.changeset/multi-source-sql-and-exclusion.md @@ -0,0 +1,12 @@ +--- +'@hyperdx/app': minor +'@hyperdx/common-utils': minor +--- + +SQL search now works across multiple sources, and a source that can't answer +the search says so. Previously multi-source search refused SQL outright, and a +Lucene query naming a field a source doesn't have silently returned nothing +from that source — the field resolved to a condition matching no rows. Both +languages now resolve the columns a search references against each source's +schema up front: sources that have them run the query, and a source that +doesn't is excluded with the missing column named on its status chip. diff --git a/packages/app/src/DBSearchPage.tsx b/packages/app/src/DBSearchPage.tsx index caa99bf8df..49bfcfbea0 100644 --- a/packages/app/src/DBSearchPage.tsx +++ b/packages/app/src/DBSearchPage.tsx @@ -29,6 +29,7 @@ import HyperDX from '@hyperdx/browser'; import { ClickHouseQueryError, ColumnMeta, + extractColumnReferencesFromKey, } from '@hyperdx/common-utils/dist/clickhouse'; import { tcFromSource } from '@hyperdx/common-utils/dist/core/metadata'; import { @@ -41,6 +42,7 @@ import { isBrowser, splitAndTrimWithBracket, } from '@hyperdx/common-utils/dist/core/utils'; +import { extractFieldsFromLucene } from '@hyperdx/common-utils/dist/queryParser'; import { BuilderChartConfigWithDateRange, ChartConfigWithDateRange, @@ -123,6 +125,7 @@ import { useAliasMapFromChartConfig } from '@/hooks/useChartConfig'; import { useExplainQuery } from '@/hooks/useExplainQuery'; import { resolveExtraColumnsForSource, + unresolvedColumns, unresolvedFilterColumns, useMultiSourceColumns, } from '@/hooks/useMultiSourceSearch'; @@ -1371,13 +1374,8 @@ export function DBSearchPage() { const isMultiSource = searchedMultiSources.length > 1; // Delta/pattern analyses are per-source; multi mode pins the results view. const effectiveAnalysisMode = isMultiSource ? 'results' : analysisMode; - // Raw SQL WHERE names concrete columns of a concrete table — reinterpreting - // it per source risks silently-wrong results, so multi mode requires Lucene. - const isMultiSourceSqlBlocked = - isMultiSource && - (searchedConfig.whereLanguage ?? getStoredLanguage() ?? 'lucene') === - 'sql' && - !!searchedConfig.where; + const searchedWhereLanguage = + searchedConfig.whereLanguage ?? getStoredLanguage() ?? 'lucene'; // A hand-authored URL may carry only ?sources=; backfill the primary so the // single-source machinery (form, chart config) has one. @@ -1695,18 +1693,36 @@ export function DBSearchPage() { () => (isMultiSource ? (searchedConfig.filters ?? []) : []), [isMultiSource, searchedConfig.filters], ); + // Columns the search itself names, so a source that lacks one can be + // excluded with a reason instead of erroring (SQL) or silently matching + // nothing (Lucene resolves an unknown field to `1 = 0`). A query we can't + // parse yields null, which excludes nobody. + const queryColumnRefs = useMemo(() => { + const where = searchedConfig.where ?? ''; + if (!isMultiSource || !where.trim()) return []; + return searchedWhereLanguage === 'sql' + ? extractColumnReferencesFromKey(where) + : (extractFieldsFromLucene(where) ?? []); + }, [isMultiSource, searchedConfig.where, searchedWhereLanguage]); + const multiDisabledReasons = useMemo(() => { const reasons = new Map(); - if (!isMultiSource || multiSourceFilters.length === 0) return reasons; + if (!isMultiSource) return reasons; + if (multiSourceFilters.length === 0 && queryColumnRefs.length === 0) { + return reasons; + } for (const source of searchedMultiSources) { - const missing = unresolvedFilterColumns( + const sourceColumns = columnsBySourceId.get(source.id); + const missingInQuery = unresolvedColumns(queryColumnRefs, sourceColumns); + const missingInFilters = unresolvedFilterColumns( multiSourceFilters, - columnsBySourceId.get(source.id), + sourceColumns, ); + const missing = [...new Set([...missingInQuery, ...missingInFilters])]; if (missing.length > 0) { reasons.set( source.id, - `${source.name} is excluded: the active filter uses ${missing.join( + `${source.name} is excluded: this search uses ${missing.join( ', ', )}, which it doesn't have`, ); @@ -1716,6 +1732,7 @@ export function DBSearchPage() { }, [ isMultiSource, multiSourceFilters, + queryColumnRefs, searchedMultiSources, columnsBySourceId, ]); @@ -1741,7 +1758,6 @@ export function DBSearchPage() { if (dbSqlRowTableConfig == null || searchedSource == null) return []; return [{ source: searchedSource, config: dbSqlRowTableConfig }]; } - if (isMultiSourceSqlBlocked) return []; // Extra columns and filters both need each source's DESCRIBE (to resolve // column-vs-NULL and filter resolvability); hold the row queries until // they've loaded so we don't fire throwaway or erroring queries. @@ -1761,7 +1777,7 @@ export function DBSearchPage() { source, { where, - whereLanguage: 'lucene', + whereLanguage: searchedWhereLanguage, filters: multiSourceFilters, orderBy: multiSourceDefaultOrderBy(source), }, @@ -1778,11 +1794,11 @@ export function DBSearchPage() { })); }, [ isMultiSource, - isMultiSourceSqlBlocked, searchedMultiSources, searchedSource, dbSqlRowTableConfig, searchedConfig.where, + searchedWhereLanguage, multiExtraColumnNames, multiSourceFilters, multiDisabledReasons, @@ -1791,7 +1807,7 @@ export function DBSearchPage() { ]); const multiHistogramSpecs = useMemo(() => { - if (!isMultiSource || isMultiSourceSqlBlocked) return []; + if (!isMultiSource) return []; if ( multiSourceFilters.length > 0 && columnsBySourceId.size < searchedMultiSources.length @@ -1805,7 +1821,7 @@ export function DBSearchPage() { config: { ...buildMultiSourceSearchConfig(source, { where, - whereLanguage: 'lucene', + whereLanguage: searchedWhereLanguage, filters: multiSourceFilters, }), select: ALERT_COUNT_DEFAULT_SELECT, @@ -1821,9 +1837,9 @@ export function DBSearchPage() { })); }, [ isMultiSource, - isMultiSourceSqlBlocked, searchedMultiSources, searchedConfig.where, + searchedWhereLanguage, multiSourceFilters, multiDisabledReasons, columnsBySourceId, @@ -2971,84 +2987,71 @@ export function DBSearchPage() { )} {effectiveAnalysisMode === 'results' && isMultiSource && ( - {isMultiSourceSqlBlocked ? ( - - - SQL search isn't supported across multiple sources - - - A SQL WHERE clause references the columns of one - specific table. Switch the search language to Lucene to - search across sources, or go back to a single source. - - - ) : ( - <> - - - - {isFilterSidebarCollapsed && ( - - setIsFilterSidebarCollapsed(false) - } - /> - )} - - - {shouldShowLiveModeHint && ( - + + + + {isFilterSidebarCollapsed && ( + + setIsFilterSidebarCollapsed(false) + } /> )} + - - - - - - - - - )} + {shouldShowLiveModeHint && ( + + )} + + + + + + + + + )} {effectiveAnalysisMode === 'results' && !isMultiSource && ( diff --git a/packages/app/src/hooks/__tests__/useMultiSourceSearch.test.ts b/packages/app/src/hooks/__tests__/useMultiSourceSearch.test.ts index 58be8d33d2..3fa3d6ea63 100644 --- a/packages/app/src/hooks/__tests__/useMultiSourceSearch.test.ts +++ b/packages/app/src/hooks/__tests__/useMultiSourceSearch.test.ts @@ -3,6 +3,8 @@ import { Filter } from '@hyperdx/common-utils/dist/types'; import { filterRootColumn, resolveExtraColumnsForSource, + rootColumnOf, + unresolvedColumns, unresolvedFilterColumns, } from '@/hooks/useMultiSourceSearch'; @@ -85,3 +87,56 @@ describe('resolveExtraColumnsForSource', () => { ]); }); }); + +describe('rootColumnOf', () => { + it('returns a plain column unchanged', () => { + expect(rootColumnOf('ServiceName')).toBe('ServiceName'); + }); + + it('returns the root of a map subscript or JSON path', () => { + expect(rootColumnOf("LogAttributes['level']")).toBe('LogAttributes'); + expect(rootColumnOf('LogAttributes.level')).toBe('LogAttributes'); + }); + + it('unquotes a backticked identifier', () => { + expect(rootColumnOf('`weird-col`')).toBe('weird-col'); + }); + + it('returns null for something not rooted at a column', () => { + expect(rootColumnOf("'a literal'")).toBeNull(); + expect(rootColumnOf('123')).toBeNull(); + }); +}); + +describe('unresolvedColumns', () => { + const columns = new Set(['ServiceName', 'Body', 'LogAttributes']); + + it('reports references the source lacks', () => { + expect(unresolvedColumns(['ServiceName', 'StatusCode'], columns)).toEqual([ + 'StatusCode', + ]); + }); + + it('resolves map and JSON references through their root column', () => { + expect( + unresolvedColumns( + ["LogAttributes['level']", 'SpanAttributes.http'], + columns, + ), + ).toEqual(['SpanAttributes']); + }); + + it('ignores references it cannot attribute to a column', () => { + expect(unresolvedColumns([null, "'literal'"], columns)).toEqual([]); + }); + + it('excludes nobody while the column list is still unknown', () => { + expect(unresolvedColumns(['StatusCode'], undefined)).toEqual([]); + }); + + it('dedupes a column referenced more than once', () => { + expect(unresolvedColumns(['StatusCode', 'StatusCode'], columns)).toEqual([ + 'StatusCode', + ]); + }); +}); diff --git a/packages/app/src/hooks/useMultiSourceSearch.ts b/packages/app/src/hooks/useMultiSourceSearch.ts index f4ecdb4da4..e83607b8ee 100644 --- a/packages/app/src/hooks/useMultiSourceSearch.ts +++ b/packages/app/src/hooks/useMultiSourceSearch.ts @@ -88,36 +88,50 @@ export function useMultiSourceColumns(sources: TSource[]): { }, [slotData, sources]); } +/** + * The table column a reference is rooted at: `ServiceName` for `ServiceName`, + * `LogAttributes` for `LogAttributes['level']` or `LogAttributes.level`, and + * the unquoted name for a backticked identifier. Returns null for anything + * that isn't rooted at a plain column (a bare function call, say), so callers + * treat it as "can't attribute" rather than "missing". + */ +export function rootColumnOf(reference: string): string | null { + const ref = reference.trim(); + const backticked = ref.match(/^`([^`]+)`/); + if (backticked) return backticked[1]; + const plain = ref.match(/^[A-Za-z_][A-Za-z0-9_]*/); + return plain ? plain[0] : null; +} + /** * Root column a filter references, for per-source resolvability checks. - * sql_ast filters carry the escaped SQL key in `left` (e.g. `ServiceName`, - * a backticked identifier, or `LogAttributes['level']` whose root is - * `LogAttributes`). Other filter types (raw sql/lucene conditions) can't be - * attributed to a single column and return null — callers should apply them - * to every source and rely on per-source error isolation. + * sql_ast filters carry the escaped SQL key in `left`. Other filter types + * (raw sql/lucene conditions) can't be attributed to a single column and + * return null — callers should apply them to every source and rely on + * per-source error isolation. */ export function filterRootColumn(filter: Filter): string | null { if (filter.type !== 'sql_ast') return null; - const left = filter.left.trim(); - const backticked = left.match(/^`([^`]+)`/); - if (backticked) return backticked[1]; - const plain = left.match(/^[A-Za-z_][A-Za-z0-9_]*/); - return plain ? plain[0] : null; + return rootColumnOf(filter.left); } /** - * For one source: which of the active filters reference a column its table - * doesn't have. A non-empty result means the source can't answer the - * filtered search and should be excluded (with a visible reason). + * For one source: which of the given column references its table doesn't + * have. A non-empty result means the source can't answer the search and + * should be excluded, with the reason shown to the user. + * + * Unknown columns (null root) and an unknown column list are both treated as + * "no reason to exclude" — better to run the query and surface a real error + * than to drop a source on a guess. */ -export function unresolvedFilterColumns( - filters: Filter[], +export function unresolvedColumns( + references: (string | null)[], sourceColumns: Set | undefined, ): string[] { if (sourceColumns == null) return []; const missing = new Set(); - for (const filter of filters) { - const root = filterRootColumn(filter); + for (const reference of references) { + const root = reference == null ? null : rootColumnOf(reference); if (root != null && !sourceColumns.has(root)) { missing.add(root); } @@ -125,6 +139,17 @@ export function unresolvedFilterColumns( return [...missing]; } +/** + * For one source: which columns the active filters reference that its table + * doesn't have. + */ +export function unresolvedFilterColumns( + filters: Filter[], + sourceColumns: Set | undefined, +): string[] { + return unresolvedColumns(filters.map(filterRootColumn), sourceColumns); +} + /** Quote a column name as a ClickHouse identifier when it needs it. */ function quoteIdentifier(name: string): string { return /^[A-Za-z_][A-Za-z0-9_]*$/.test(name) diff --git a/packages/common-utils/src/__tests__/queryParser.test.ts b/packages/common-utils/src/__tests__/queryParser.test.ts index ddfd1f7764..3a90bcb9b5 100644 --- a/packages/common-utils/src/__tests__/queryParser.test.ts +++ b/packages/common-utils/src/__tests__/queryParser.test.ts @@ -3,6 +3,7 @@ import { ClickhouseClient } from '@/clickhouse/node'; import { getMetadata } from '@/core/metadata'; import { CustomSchemaSQLSerializerV2, + extractFieldsFromLucene, genEnglishExplanation, parseKvItemsCastExpression, parseKvItemsExpression, @@ -3192,3 +3193,60 @@ describe('CustomSchemaSQLSerializerV2 - KV items version gate', () => { expect(sql).toContain(HAS_FORM); }); }); + +describe('extractFieldsFromLucene', () => { + it('returns no fields for an empty query', () => { + expect(extractFieldsFromLucene('')).toEqual([]); + expect(extractFieldsFromLucene(' ')).toEqual([]); + }); + + it("ignores bare terms, which match a source's implicit column", () => { + expect(extractFieldsFromLucene('error')).toEqual([]); + expect(extractFieldsFromLucene('"connection refused"')).toEqual([]); + }); + + it('collects a single field', () => { + expect(extractFieldsFromLucene('ServiceName:cart')).toEqual([ + 'ServiceName', + ]); + }); + + it('collects fields across boolean operators and groups', () => { + expect( + extractFieldsFromLucene( + 'ServiceName:cart AND (StatusCode:Error OR SeverityText:warn)', + ).sort(), + ).toEqual(['ServiceName', 'SeverityText', 'StatusCode']); + }); + + it('collects the field of a negated term without the leading dash', () => { + expect(extractFieldsFromLucene('-ServiceName:cart')).toEqual([ + 'ServiceName', + ]); + }); + + it('collects nested and ranged fields', () => { + expect(extractFieldsFromLucene('LogAttributes.level:error')).toEqual([ + 'LogAttributes.level', + ]); + expect(extractFieldsFromLucene('Duration:[100 TO 200]')).toEqual([ + 'Duration', + ]); + }); + + it('dedupes a field used more than once', () => { + expect( + extractFieldsFromLucene('ServiceName:cart OR ServiceName:checkout'), + ).toEqual(['ServiceName']); + }); + + it('returns null when the query cannot be parsed', () => { + expect(extractFieldsFromLucene('ServiceName:(')).toBeNull(); + }); + + it('mixes bare terms with fielded terms', () => { + expect(extractFieldsFromLucene('timeout ServiceName:cart')).toEqual([ + 'ServiceName', + ]); + }); +}); diff --git a/packages/common-utils/src/queryParser.ts b/packages/common-utils/src/queryParser.ts index d6c76ea314..dee1068288 100644 --- a/packages/common-utils/src/queryParser.ts +++ b/packages/common-utils/src/queryParser.ts @@ -2126,6 +2126,53 @@ async function serialize( return ''; } +/** + * The field names a Lucene query filters on, e.g. `["ServiceName"]` for + * `ServiceName:cart AND error`. + * + * Bare terms are excluded: they match against whichever column a source + * designates as its implicit/body column, so they are always resolvable. + * Returns null when the query can't be parsed, so callers can't mistake + * "unparseable" for "references nothing". + */ +export function extractFieldsFromLucene(query: string): string[] | null { + if (!query.trim()) return []; + + let ast: lucene.AST; + try { + ast = parse(query); + } catch { + return null; + } + + const fields = new Set(); + const visit = (node: lucene.AST | lucene.Node | undefined) => { + if (node == null) return; + + if (isNodeTerm(node) || isNodeRangedTerm(node)) { + const raw = node.field; + if (raw != null && raw !== IMPLICIT_FIELD) { + // A leading `-` negates the term rather than naming the column. + fields.add(raw[0] === '-' ? raw.slice(1) : raw); + } + return; + } + + if (isBinaryAST(node)) { + visit(node.left); + visit(node.right); + return; + } + + if (isLeftOnlyAST(node)) { + visit(node.left); + } + }; + visit(ast); + + return [...fields]; +} + // TODO: can just inline this within getSearchQuery export async function genWhereSQL( ast: lucene.AST, From c64b8a2f4a22be35faddf4153546991dceb02f5d Mon Sep 17 00:00:00 2001 From: Tom Alexander Date: Wed, 12 Aug 2026 16:36:29 -0400 Subject: [PATCH 8/8] test(common-utils): pin per-source WHERE rendering for both languages MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Renders a multi-source search's per-source queries against a mocked metadata layer and asserts the user's condition reaches each one: a SQL condition passes through verbatim to both tables, a Lucene field resolves against each source, and a bare Lucene term lands on each source's own implicit column (Body for logs, SpanName for traces). This is the check the SQL path had been missing — the language used to be refused in multi-source search, so nothing exercised it end to end. --- .../src/__tests__/multiSourceWhere.test.ts | 137 ++++++++++++++++++ .../src/__tests__/queryParser.test.ts | 2 +- 2 files changed, 138 insertions(+), 1 deletion(-) create mode 100644 packages/common-utils/src/__tests__/multiSourceWhere.test.ts diff --git a/packages/common-utils/src/__tests__/multiSourceWhere.test.ts b/packages/common-utils/src/__tests__/multiSourceWhere.test.ts new file mode 100644 index 0000000000..08818e4b78 --- /dev/null +++ b/packages/common-utils/src/__tests__/multiSourceWhere.test.ts @@ -0,0 +1,137 @@ +import { parameterizedQueryToSql } from '@/clickhouse'; +import { Metadata } from '@/core/metadata'; +import { renderChartConfig } from '@/core/renderChartConfig'; +import { buildMultiSourceSearchConfig } from '@/core/searchChartConfig'; +import { SourceKind, TSource } from '@/types'; + +/** + * A search spanning several sources renders one query per source. These tests + * pin down that the user's WHERE reaches each source's query in both + * languages, since a SQL condition is passed through verbatim while a Lucene + * one is resolved against each source's own schema. + */ +describe('multi-source WHERE rendering', () => { + let metadata: jest.Mocked; + + beforeEach(() => { + metadata = { + getColumns: jest.fn().mockResolvedValue([ + { name: 'Timestamp', type: 'DateTime64(9)' }, + { name: 'Body', type: 'String' }, + { name: 'ServiceName', type: 'String' }, + { name: 'SeverityText', type: 'String' }, + ]), + // The real implementation always returns a Map (possibly empty). + getMaterializedColumnsLookupTable: jest.fn().mockResolvedValue(new Map()), + getColumn: jest + .fn() + .mockImplementation(async ({ column }: { column: string }) => + column === 'ServiceName' || column === 'Body' + ? { name: column, type: 'String' } + : undefined, + ), + getTableMetadata: jest + .fn() + .mockResolvedValue({ primary_key: 'Timestamp' }), + getSkipIndices: jest.fn().mockResolvedValue([]), + getSetting: jest.fn().mockResolvedValue(undefined), + isClickHouseCloud: jest.fn().mockResolvedValue(false), + } as unknown as jest.Mocked; + }); + + const logSource = { + id: 'logs', + kind: SourceKind.Log, + name: 'Logs', + connection: 'conn-1', + from: { databaseName: 'default', tableName: 'otel_logs' }, + timestampValueExpression: 'Timestamp', + defaultTableSelectExpression: 'Timestamp, Body', + implicitColumnExpression: 'Body', + bodyExpression: 'Body', + serviceNameExpression: 'ServiceName', + severityTextExpression: 'SeverityText', + } as unknown as TSource; + + const traceSource = { + id: 'traces', + kind: SourceKind.Trace, + name: 'Traces', + connection: 'conn-2', + from: { databaseName: 'default', tableName: 'otel_traces' }, + timestampValueExpression: 'Timestamp', + defaultTableSelectExpression: 'Timestamp, SpanName', + implicitColumnExpression: 'SpanName', + serviceNameExpression: 'ServiceName', + statusCodeExpression: 'StatusCode', + spanNameExpression: 'SpanName', + durationExpression: 'Duration', + durationPrecision: 9, + } as unknown as TSource; + + const dateRange: [Date, Date] = [ + new Date('2026-01-01T00:00:00Z'), + new Date('2026-01-01T01:00:00Z'), + ]; + + const render = async ( + source: TSource, + where: string, + language: 'sql' | 'lucene', + ) => + parameterizedQueryToSql( + await renderChartConfig( + { + ...buildMultiSourceSearchConfig(source, { + where, + whereLanguage: language, + orderBy: 'Timestamp DESC', + }), + dateRange, + }, + metadata, + undefined, + ), + ); + + it('passes a SQL condition through to each source, against its own table', async () => { + const where = "ServiceName = 'cart'"; + + const logSql = await render(logSource, where, 'sql'); + const traceSql = await render(traceSource, where, 'sql'); + + expect(logSql).toContain("ServiceName = 'cart'"); + expect(traceSql).toContain("ServiceName = 'cart'"); + // Each query targets its own table and projects the shared aliases. + expect(logSql).toContain('otel_logs'); + expect(traceSql).toContain('otel_traces'); + expect(logSql).toContain('__hdx_timestamp'); + expect(traceSql).toContain('__hdx_timestamp'); + }); + + it('resolves a Lucene field against each source and keeps them independent', async () => { + const logSql = await render(logSource, 'ServiceName:cart', 'lucene'); + const traceSql = await render(traceSource, 'ServiceName:cart', 'lucene'); + + expect(logSql).toContain('ServiceName'); + expect(traceSql).toContain('ServiceName'); + expect(logSql).toContain('otel_logs'); + expect(traceSql).toContain('otel_traces'); + }); + + it('resolves a bare Lucene term against each source implicit column', async () => { + const logSql = await render(logSource, 'timeout', 'lucene'); + const traceSql = await render(traceSource, 'timeout', 'lucene'); + + // Logs search their body column; traces search the span name. + expect(logSql).toContain('Body'); + expect(traceSql).toContain('SpanName'); + }); + + it('renders each source against its own connection and timestamp bounds', async () => { + const logSql = await render(logSource, "ServiceName = 'cart'", 'sql'); + + expect(logSql).toContain('Timestamp'); + expect(logSql).toContain('ORDER BY'); + }); +}); diff --git a/packages/common-utils/src/__tests__/queryParser.test.ts b/packages/common-utils/src/__tests__/queryParser.test.ts index 3a90bcb9b5..43780bc2b1 100644 --- a/packages/common-utils/src/__tests__/queryParser.test.ts +++ b/packages/common-utils/src/__tests__/queryParser.test.ts @@ -3215,7 +3215,7 @@ describe('extractFieldsFromLucene', () => { expect( extractFieldsFromLucene( 'ServiceName:cart AND (StatusCode:Error OR SeverityText:warn)', - ).sort(), + )?.sort(), ).toEqual(['ServiceName', 'SeverityText', 'StatusCode']); });