From f3e60aeac0fde1bd3d0e090655fe29a8c4d7c694 Mon Sep 17 00:00:00 2001 From: Tom Alexander Date: Tue, 11 Aug 2026 09:16:58 -0400 Subject: [PATCH 1/3] feat(app): search across multiple log and trace sources at once The search page's source selector can expand into a multi-select (up to 5 log/trace sources). Each selected source runs its own query pipeline (own connection, Lucene serializer, windowed pagination) with its SELECT rewritten to canonical aliases; a client-side k-way merge interleaves the streams by timestamp behind a "safe frontier" so the timeline never shows a gap a lagging source could still fill. Results render in one table with per-row source badges, a histogram stacked by source, summed totals, and an add-column picker over the union of the sources' columns. Multi mode is Lucene-only and URL-shareable (?sources=). Saved searches, alerts, filters, and delta/pattern modes stay single-source and are gated off with explanations; single-source behavior is unchanged. Also fixes chSqlToAliasMap dropping NULL-literal projections, which would have made row-WHERE clauses reference nonexistent columns. --- .changeset/multi-source-search.md | 15 + packages/app/src/DBSearchPage.tsx | 555 +++++++++++++++--- .../DBSearchPage.directTrace.test.tsx | 7 + packages/app/src/components/DBRowTable.tsx | 27 + .../components/DBSqlRowTableWithSidebar.tsx | 4 +- .../app/src/components/MultiSourceBadge.tsx | 34 ++ .../components/MultiSourceColumnPicker.tsx | 71 +++ .../src/components/MultiSourceRowTable.tsx | 472 +++++++++++++++ .../src/components/MultiSourceTimeChart.tsx | 320 ++++++++++ packages/app/src/defaults.ts | 8 + .../app/src/hooks/useMultiSourceSearch.ts | 91 +++ .../app/src/hooks/useOffsetPaginatedQuery.tsx | 8 +- .../app/src/hooks/useResolvedSourcesParam.ts | 58 ++ .../utils/__tests__/multiSourceMerge.test.ts | 379 ++++++++++++ packages/app/src/utils/multiSourceMerge.ts | 239 ++++++++ packages/app/src/utils/sourceParams.ts | 45 ++ .../src/__tests__/clickhouse.test.ts | 16 + packages/common-utils/src/clickhouse/index.ts | 4 + .../core/__tests__/searchChartConfig.test.ts | 109 ++++ .../src/core/searchChartConfig.ts | 141 +++++ 20 files changed, 2523 insertions(+), 80 deletions(-) create mode 100644 .changeset/multi-source-search.md create mode 100644 packages/app/src/components/MultiSourceBadge.tsx create mode 100644 packages/app/src/components/MultiSourceColumnPicker.tsx create mode 100644 packages/app/src/components/MultiSourceRowTable.tsx create mode 100644 packages/app/src/components/MultiSourceTimeChart.tsx create mode 100644 packages/app/src/hooks/useMultiSourceSearch.ts create mode 100644 packages/app/src/hooks/useResolvedSourcesParam.ts create mode 100644 packages/app/src/utils/__tests__/multiSourceMerge.test.ts create mode 100644 packages/app/src/utils/multiSourceMerge.ts diff --git a/.changeset/multi-source-search.md b/.changeset/multi-source-search.md new file mode 100644 index 0000000000..9af499a0b7 --- /dev/null +++ b/.changeset/multi-source-search.md @@ -0,0 +1,15 @@ +--- +'@hyperdx/app': minor +'@hyperdx/common-utils': minor +--- + +Search across multiple sources at once. The search page's source selector can +now expand into a multi-select (up to 5 log/trace sources): results interleave +into one timestamp-ordered timeline with a per-row source badge, normalized +columns (Timestamp, Source, Service, Level, Message, and Duration when traces +are included), a histogram stacked by source, and an add-column picker over the +union of the selected sources' columns. Each source runs its own query +pipeline — sources on different connections work, and a failing source shows a +status chip instead of failing the whole search. Multi-source mode is +Lucene-only and shareable via URL; saved searches and alerts remain +single-source for now. diff --git a/packages/app/src/DBSearchPage.tsx b/packages/app/src/DBSearchPage.tsx index cdae127d1e..acd284bbed 100644 --- a/packages/app/src/DBSearchPage.tsx +++ b/packages/app/src/DBSearchPage.tsx @@ -31,7 +31,11 @@ import { ColumnMeta, } from '@hyperdx/common-utils/dist/clickhouse'; import { tcFromSource } from '@hyperdx/common-utils/dist/core/metadata'; -import { buildSearchChartConfig } from '@hyperdx/common-utils/dist/core/searchChartConfig'; +import { + ALERT_COUNT_DEFAULT_SELECT, + buildMultiSourceSearchConfig, + buildSearchChartConfig, +} from '@hyperdx/common-utils/dist/core/searchChartConfig'; import { aliasMapToWithClauses, isBrowser, @@ -95,21 +99,34 @@ import { ErrorBoundary } from '@/components/Error/ErrorBoundary'; import { FavoriteButton } from '@/components/FavoriteButton'; import ResourceTerraformPopover from '@/components/Iac/ResourceTerraformPopover'; import { InputControlled } from '@/components/InputControlled'; +import MultiSourceColumnPicker from '@/components/MultiSourceColumnPicker'; +import MultiSourceRowTableWithSidebar from '@/components/MultiSourceRowTable'; +import { + MultiSourceTimeChart, + MultiSourceTotalCountChart, +} from '@/components/MultiSourceTimeChart'; import OnboardingModal from '@/components/OnboardingModal'; import SearchWhereInput, { getStoredLanguage, } from '@/components/SearchInput/SearchWhereInput'; import SearchPageActionBar from '@/components/SearchPageActionBar'; import SearchTotalCountChart from '@/components/SearchTotalCountChart'; +import { SourceMultiSelectControlled } from '@/components/SourceMultiSelect'; import { TableSourceForm } from '@/components/Sources/SourceForm'; import { SourceSelectControlled } from '@/components/SourceSelect'; import { SQLInlineEditorControlled } from '@/components/SQLEditor/SQLInlineEditor'; import { Tags } from '@/components/Tags'; import { TimePicker } from '@/components/TimePicker'; import { IS_LOCAL_MODE } from '@/config'; +import { MAX_SEARCH_SOURCES } from '@/defaults'; import { useAliasMapFromChartConfig } from '@/hooks/useChartConfig'; import { useExplainQuery } from '@/hooks/useExplainQuery'; +import { + resolveExtraColumnsForSource, + useMultiSourceColumns, +} from '@/hooks/useMultiSourceSearch'; import { useResolvedSourceParam } from '@/hooks/useResolvedSourceParam'; +import { useResolvedSourcesParam } from '@/hooks/useResolvedSourcesParam'; import { withAppNav } from '@/layout'; import { useCreateSavedSearch, @@ -181,6 +198,10 @@ const ALLOWED_SOURCE_KINDS = [SourceKind.Log, SourceKind.Trace]; const SearchConfigSchema = z.object({ select: z.string(), source: z.string(), + // Multi-source search: the full selection (2+ engages multi mode). The + // single `source` field stays the primary (= sources[0]) so every + // single-source code path keeps working unchanged. + sources: z.array(z.string()), where: z.string(), whereLanguage: z.enum(['sql', 'lucene']), orderBy: z.string(), @@ -865,6 +886,25 @@ function optimizeDefaultOrderBy( : `${orderByArr[0]} DESC`; } +/** + * Per-source default ORDER BY for multi-source search. Same resolution as + * useDefaultOrderBy minus the table sorting-key optimization (which needs a + * metadata query per source): the source's explicit orderByExpression, else + * its timestamp expression(s) DESC. Time-window pagination requires the first + * term to be the source's timestamp, which this guarantees. + */ +function multiSourceDefaultOrderBy(source: TSource): string { + const isEventSource = + source.kind === SourceKind.Log || source.kind === SourceKind.Trace; + const explicit = isEventSource ? source.orderByExpression?.trim() : undefined; + if (explicit) return explicit; + return optimizeDefaultOrderBy( + source.timestampValueExpression ?? '', + isEventSource ? source.displayedTimestampValueExpression : undefined, + undefined, + ); +} + export function useDefaultOrderBy(sourceID: string | undefined | null) { const { data: source } = useSource({ id: sourceID, @@ -895,6 +935,9 @@ function formatDroppedFiltersMessage(count: number): string { // This is outside as it needs to be a stable reference const queryStateMap = { source: parseAsString, + // JSON-encoded (not comma-separated) because source names may contain + // commas; `source` is always written alongside it as the primary. + sources: parseAsJsonEncoded(), where: parseAsStringEncoded, select: parseAsStringEncoded, whereLanguage: parseAsStringEnum<'sql' | 'lucene'>(['sql', 'lucene']), @@ -1116,6 +1159,7 @@ export function DBSearchPage() { (savedSearchId || directTraceId || rawSearchedConfig.source ? '' : defaultSourceId), + sources: searchedConfig.sources ?? [], filters: searchedConfig.filters ?? [], orderBy: searchedConfig.orderBy ?? '', }, @@ -1184,6 +1228,7 @@ export function DBSearchPage() { whereLanguage: searchedConfig?.whereLanguage ?? getStoredLanguage() ?? 'lucene', source: searchedConfig?.source ?? undefined, + sources: searchedConfig?.sources ?? [], filters: searchedConfig?.filters ?? [], orderBy: searchedConfig?.orderBy ?? '', }); @@ -1198,6 +1243,7 @@ export function DBSearchPage() { // to an existing source. const isSearchConfigEmpty = !rawSearchedConfig.source && + !rawSearchedConfig.sources?.length && !where && !select && !whereLanguage && @@ -1240,6 +1286,7 @@ export function DBSearchPage() { savedSearch, searchedConfig, rawSearchedConfig.source, + rawSearchedConfig.sources, setSearchedConfig, savedSearchId, defaultSourceId, @@ -1268,12 +1315,15 @@ export function DBSearchPage() { const onSubmit = useCallback(() => { onSearch(displayedTimeInputValue); handleSubmit( - ({ select, where, whereLanguage, source, filters, orderBy }) => { + ({ select, where, whereLanguage, source, sources, filters, orderBy }) => { setSearchedConfig({ select, where, whereLanguage, source, + // Writer discipline: only 2+ selections persist the list; a single + // selection clears it so old-style URLs stay canonical. + sources: sources.length > 1 ? sources : null, filters, orderBy, }); @@ -1489,6 +1539,202 @@ export function DBSearchPage() { const { data: chartConfig, isLoading: isChartConfigLoading } = useSearchedConfigToChartConfig(chartSearchConfig, defaultSearchConfig); + // --- Multi-source search ------------------------------------------------- + // 2+ resolved sources in the ?sources= param engage multi mode: one + // independent query pipeline per source, merged client-side. The single + // `source` (primary) keeps every existing code path working; multi mode + // only swaps what gets rendered below. + const { sources: searchedMultiSources } = useResolvedSourcesParam( + rawSearchedConfig.sources, + { kinds: ALLOWED_SOURCE_KINDS }, + ); + const isMultiSource = searchedMultiSources.length > 1; + // Delta/pattern analyses are per-source; multi mode pins the results view. + const effectiveAnalysisMode = isMultiSource ? 'results' : analysisMode; + // Raw SQL WHERE names concrete columns of a concrete table — reinterpreting + // it per source risks silently-wrong results, so multi mode requires Lucene. + const isMultiSourceSqlBlocked = + isMultiSource && + (searchedConfig.whereLanguage ?? getStoredLanguage() ?? 'lucene') === + 'sql' && + !!searchedConfig.where; + + // A hand-authored URL may carry only ?sources=; backfill the primary so the + // single-source machinery (form, chart config) has one. + useEffect(() => { + if (!rawSearchedConfig.source && searchedMultiSources.length > 0) { + setSearchedConfig({ source: searchedMultiSources[0].id }); + } + }, [rawSearchedConfig.source, searchedMultiSources, setSearchedConfig]); + + const watchedSources = useWatch({ control, name: 'sources' }); + const formSourceCount = watchedSources?.length ?? 0; + // The multi-select UI stays visible while the user is composing a selection + // (even before a second source is added). + const [multiPickerOpen, setMultiPickerOpen] = useState(false); + const isMultiSelectUI = multiPickerOpen || formSourceCount > 1; + const formIsMulti = formSourceCount > 1; + + const enterMultiSourceSelect = useCallback(() => { + setValue('sources', watchedSource ? [watchedSource] : []); + setMultiPickerOpen(true); + }, [setValue, watchedSource]); + + // Keep the primary `source` field in sync with the selection and re-run the + // search when the selection changes. + const prevWatchedSourcesRef = useRef(null); + useEffect(() => { + const current = watchedSources ?? []; + const prev = prevWatchedSourcesRef.current; + if (prev != null && JSON.stringify(prev) === JSON.stringify(current)) { + return; + } + prevWatchedSourcesRef.current = current; + if (prev == null) { + // Initial hydration from the URL — nothing changed. + return; + } + if (current.length > 0 && current[0] !== watchedSource) { + setValue('source', current[0]); + } + if ((prev?.length ?? 0) <= 1 && current.length > 1) { + // Entering multi mode: the single-source SELECT/ORDER BY strings don't + // translate to the canonical multi-source shape. + setValue('select', ''); + setValue('orderBy', ''); + } + debouncedSubmit(); + }, [watchedSources, watchedSource, setValue, debouncedSubmit]); + + // Collapse the picker back to the single-source select when the user + // reduces a real multi selection to one source. Keyed on the >1 → ≤1 + // transition so it can't fire in the just-opened composing state (picker + // open, one source selected, second not yet picked). + const prevFormSourceCountRef = useRef(formSourceCount); + useEffect(() => { + const prev = prevFormSourceCountRef.current; + prevFormSourceCountRef.current = formSourceCount; + if (prev > 1 && formSourceCount <= 1) { + setMultiPickerOpen(false); + } + }, [formSourceCount]); + + // Per-source top-level columns: powers the add-column picker and the + // per-source `column vs NULL` projection for user-picked extras. Driven by + // the form's draft selection while composing (so the picker has options + // before the search is submitted), falling back to the searched selection. + const formMultiSources = useMemo( + () => + (watchedSources ?? []) + .map(id => inputSourceObjs?.find(s => s.id === id)) + .filter((s): s is TSource => s != null), + [watchedSources, inputSourceObjs], + ); + const { columnsBySourceId, unionColumns } = useMultiSourceColumns( + formMultiSources.length > 1 + ? formMultiSources + : isMultiSource + ? searchedMultiSources + : [], + ); + + // In multi mode the `select` param holds the extra column names picked by + // the user (the canonical columns are always projected). + const multiExtraColumnNames = useMemo( + () => + isMultiSource ? splitAndTrimWithBracket(searchedConfig.select ?? '') : [], + [isMultiSource, searchedConfig.select], + ); + + // The add-column picker edits the form's draft select (like the SELECT + // editor it replaces), then auto-submits. + const inputSelect = useWatch({ name: 'select', control }); + const multiPickerValue = useMemo( + () => (formIsMulti ? splitAndTrimWithBracket(inputSelect ?? '') : []), + [formIsMulti, inputSelect], + ); + const onMultiColumnsChange = useCallback( + (columns: string[]) => { + setValue('select', columns.join(', ')); + debouncedSubmit(); + }, + [setValue, debouncedSubmit], + ); + + const multiSourceStreamSpecs = useMemo(() => { + if (!isMultiSource || isMultiSourceSqlBlocked) return []; + // Extra columns need each source's DESCRIBE to resolve column-vs-NULL; + // hold the row queries until they've loaded so we don't fire a throwaway + // query per source with every extra projected as NULL. + if ( + multiExtraColumnNames.length > 0 && + columnsBySourceId.size < searchedMultiSources.length + ) { + return []; + } + const includeDuration = searchedMultiSources.some(isTraceSource); + const where = searchedConfig.where ?? ''; + return searchedMultiSources.map(source => ({ + source, + config: { + ...buildMultiSourceSearchConfig( + source, + { + where, + whereLanguage: 'lucene', + orderBy: multiSourceDefaultOrderBy(source), + }, + { + includeDuration, + extraColumns: resolveExtraColumnsForSource( + multiExtraColumnNames, + columnsBySourceId.get(source.id), + ), + }, + ), + dateRange: searchedTimeRange, + }, + })); + }, [ + isMultiSource, + isMultiSourceSqlBlocked, + searchedMultiSources, + searchedConfig.where, + multiExtraColumnNames, + columnsBySourceId, + searchedTimeRange, + ]); + + const multiHistogramSpecs = useMemo(() => { + if (!isMultiSource || isMultiSourceSqlBlocked) return []; + const where = searchedConfig.where ?? ''; + return searchedMultiSources.map(source => ({ + source, + config: { + ...buildMultiSourceSearchConfig(source, { + where, + whereLanguage: 'lucene', + }), + select: ALERT_COUNT_DEFAULT_SELECT, + orderBy: undefined, + granularity: 'auto' as const, + dateRange: searchedTimeRange, + displayType: DisplayType.StackedBar, + // Match the single-source histogram: reflect the user's exact range + // so chart and table counts agree (see histogramTimeChartConfig). + alignDateRangeToGranularity: false, + dateRangeEndInclusive: true, + }, + })); + }, [ + isMultiSource, + isMultiSourceSqlBlocked, + searchedMultiSources, + searchedConfig.where, + searchedTimeRange, + ]); + // --- End multi-source search --------------------------------------------- + // query error handling const { hasQueryError, queryError } = useMemo(() => { const hasQueryError = Object.values(_queryErrors).length > 0; @@ -1662,12 +1908,15 @@ export function DBSearchPage() { }, [chartConfig, searchedTimeRange]); // Stable key for persisting column widths in localStorage. Scoped per saved - // search when one is loaded, else per source for ad-hoc searches. + // search when one is loaded, else per source (or source set) for ad-hoc + // searches. const columnSizeTableId = savedSearchId ? `db-search-saved-${savedSearchId}` - : searchedConfig.source - ? `db-search-source-${searchedConfig.source}` - : undefined; + : isMultiSource + ? `db-search-multi-${searchedMultiSources.map(s => s.id).join('-')}` + : searchedConfig.source + ? `db-search-source-${searchedConfig.source}` + : undefined; const displayedColumns = useMemo(() => { // `select` is typed as `string | DerivedColumn[]` upstream, but in the @@ -1969,6 +2218,20 @@ export function DBSearchPage() { ], ); + // Multi-source rows span schemas, so single-source affordances + // (add-to-search, column toggles) are omitted — the side panel hides them. + // Passing no `source` with no toggle callbacks is required: with a null + // context source, deriveRowSidePanelContextForSource treats every row as + // same-source and would leave stale single-source callbacks enabled. + const multiRowTableContext = useMemo( + () => ({ + generateSearchUrl, + isChildModalOpen: isDrawerChildModalOpen, + setChildModalOpen: setDrawerChildModalOpen, + }), + [generateSearchUrl, isDrawerChildModalOpen, setDrawerChildModalOpen], + ); + const inputSourceTableConnection = useMemo( () => tcFromSource(inputSourceObj), [inputSourceObj], @@ -2220,22 +2483,49 @@ export function DBSearchPage() { > {/* */} - setIsSourceSchemaPreviewOpen(true)} - isSchemaPreviewEnabled={isSourceSchemaPreviewEnabled( - inputSourceObj, - )} - allowedSourceKinds={ALLOWED_SOURCE_KINDS} - data-testid="source-selector" - style={{ minWidth: 150 }} - /> + {isMultiSelectUI ? ( + + ) : ( + <> + setIsSourceSchemaPreviewOpen(true)} + isSchemaPreviewEnabled={isSourceSchemaPreviewEnabled( + inputSourceObj, + )} + allowedSourceKinds={ALLOWED_SOURCE_KINDS} + data-testid="source-selector" + style={{ minWidth: 150 }} + /> + + + + + + + )} setIsSourceSchemaPreviewOpen(false)} /> - - - - + {formIsMulti ? ( + + ) : ( + + )} + {!formIsMulti && ( + + + + )} <> {!savedSearchId ? ( - + + ) : ( + + )} @@ -2346,7 +2661,7 @@ export function DBSearchPage() { setInputValue={setDisplayedTimeInputValue} onSearch={onTimePickerSearch} onRelativeSearch={onTimePickerRelativeSearch} - showLive={analysisMode === 'results'} + showLive={effectiveAnalysisMode === 'results'} isLiveMode={isLive} // Default to relative time mode if the user has made changes to interval and reloaded. defaultRelativeTimeMode={ @@ -2378,12 +2693,14 @@ export function DBSearchPage() { - + {!isMultiSource && ( + + )} {searchedConfig != null && searchedSource != null && ( - {!isFilterSidebarCollapsed && ( + {!isFilterSidebarCollapsed && !isMultiSource && ( )} - {analysisMode === 'pattern' && + {effectiveAnalysisMode === 'pattern' && histogramTimeChartConfig != null && ( @@ -2523,7 +2840,7 @@ export function DBSearchPage() { )} - {analysisMode === 'delta' && + {effectiveAnalysisMode === 'delta' && searchedSource != null && isTraceSource(searchedSource) && ( )} - {analysisMode === 'results' && ( + {effectiveAnalysisMode === 'results' && isMultiSource && ( + + {isMultiSourceSqlBlocked ? ( + + + SQL search isn't supported across multiple sources + + + A SQL WHERE clause references the columns of one + specific table. Switch the search language to Lucene to + search across sources, or go back to a single source. + + + ) : ( + <> + + + + + {(searchedConfig.filters?.length ?? 0) > 0 && ( + + Sidebar filters aren't applied when searching + multiple sources + + )} + {shouldShowLiveModeHint && ( + + )} + + + + + + + + + + + )} + + )} + {effectiveAnalysisMode === 'results' && !isMultiSource && ( {chartConfig && histogramTimeChartConfig && ( <> diff --git a/packages/app/src/__tests__/DBSearchPage.directTrace.test.tsx b/packages/app/src/__tests__/DBSearchPage.directTrace.test.tsx index d3f10d02a5..50abfd6323 100644 --- a/packages/app/src/__tests__/DBSearchPage.directTrace.test.tsx +++ b/packages/app/src/__tests__/DBSearchPage.directTrace.test.tsx @@ -211,6 +211,13 @@ jest.mock('../components/ChartSQLPreview', () => ({ SQLPreview: () =>
, })); jest.mock('../components/DBSqlRowTableWithSidebar', () => () =>
); +// Multi-source components pull in DBRowSidePanel (and its deep import graph), +// which this test isolates away just like DBSqlRowTableWithSidebar above. +jest.mock('../components/MultiSourceRowTable', () => () =>
); +jest.mock('../components/MultiSourceTimeChart', () => ({ + MultiSourceTimeChart: () =>
, + MultiSourceTotalCountChart: () =>
, +})); jest.mock('../components/PatternTable', () => () =>
); jest.mock('../components/Search/DBSearchHeatmapChart', () => ({ DBSearchHeatmapChart: () =>
, diff --git a/packages/app/src/components/DBRowTable.tsx b/packages/app/src/components/DBRowTable.tsx index 19574d849b..f0c966c698 100644 --- a/packages/app/src/components/DBRowTable.tsx +++ b/packages/app/src/components/DBRowTable.tsx @@ -103,6 +103,7 @@ import { useLocalStorage, usePrevious, } from '@/utils'; +import { MULTI_SOURCE_ROW_FIELDS } from '@/utils/multiSourceMerge'; import ChartErrorState, { ChartErrorStateVariant, @@ -124,6 +125,7 @@ import { useExpandableRows, } from './ExpandableRowTable'; import LogLevel from './LogLevel'; +import { SourceBadge } from './MultiSourceBadge'; import styles from '@styles/LogTable.module.scss'; @@ -167,6 +169,7 @@ function getResolvedColumnSize( const jsType = opts.columnTypeMap.get(column)?._type; if (jsType === JSDataType.Date) return 170; if (column === opts.logLevelColumn) return 115; + if (column === MULTI_SOURCE_ROW_FIELDS.SOURCE_NAME) return 140; return 160; } @@ -615,6 +618,30 @@ export const RawLogTable = memo( const strValue = typeof value === 'string' ? value : `${value}`; + // Multi-source search tags each merged row with its origin + // source; render it as a colored badge (color assigned by the + // merge layer, consistent with the histogram series). + if (column === MULTI_SOURCE_ROW_FIELDS.SOURCE_NAME) { + return ( + + ); + } + + // Multi-source rows project NULL where a source lacks the + // field (e.g. Duration or a picked column for log rows); show a + // quiet dash instead of the literal "null". + if ( + value == null && + info.row.original[MULTI_SOURCE_ROW_FIELDS.SOURCE_ID] != null + ) { + return —; + } + if (column === logLevelColumn) { return ; } diff --git a/packages/app/src/components/DBSqlRowTableWithSidebar.tsx b/packages/app/src/components/DBSqlRowTableWithSidebar.tsx index bd5824e372..b500296ede 100644 --- a/packages/app/src/components/DBSqlRowTableWithSidebar.tsx +++ b/packages/app/src/components/DBSqlRowTableWithSidebar.tsx @@ -157,7 +157,9 @@ enum InlineTab { ColumnValues = 'columnValues', } -function RowOverviewPanelWrapper({ +// Exported for MultiSourceRowTable, which renders the same expanded-row +// overview but resolves the source per row instead of once per table. +export function RowOverviewPanelWrapper({ source, rowId, aliasWith, diff --git a/packages/app/src/components/MultiSourceBadge.tsx b/packages/app/src/components/MultiSourceBadge.tsx new file mode 100644 index 0000000000..7d9d75798f --- /dev/null +++ b/packages/app/src/components/MultiSourceBadge.tsx @@ -0,0 +1,34 @@ +import { COLORS } from '@/utils'; + +/** + * Stable color for the Nth selected source in a multi-source search. Indexed + * by position in the selection (not hashed) so the ≤MAX_SEARCH_SOURCES badges + * never collide; the same assignment is used by the results table badge, the + * histogram series, and the per-source status chips so a source reads as one + * color everywhere on the page. + */ +export function getMultiSourceColor(index: number): string { + return COLORS[index % COLORS.length]; +} + +/** Colored-dot source label used in the merged results table. */ +export function SourceBadge({ name, color }: { name: string; color?: string }) { + return ( + + + {name} + + ); +} diff --git a/packages/app/src/components/MultiSourceColumnPicker.tsx b/packages/app/src/components/MultiSourceColumnPicker.tsx new file mode 100644 index 0000000000..55230dd479 --- /dev/null +++ b/packages/app/src/components/MultiSourceColumnPicker.tsx @@ -0,0 +1,71 @@ +import { useCallback, useMemo } from 'react'; +import { Group, MultiSelect, Text } from '@mantine/core'; + +import { MultiSourceColumnOption } from '@/hooks/useMultiSourceSearch'; + +/** + * Multi-source replacement for the free-text SELECT editor: pick extra + * columns from the union of the selected sources' top-level columns. Columns + * missing from a source render as blank cells for that source's rows. + */ +export default function MultiSourceColumnPicker({ + unionColumns, + totalSources, + value, + onChange, +}: { + unionColumns: MultiSourceColumnOption[]; + totalSources: number; + /** Currently selected extra column names. */ + value: string[]; + onChange: (columns: string[]) => void; +}) { + const availabilityByName = useMemo( + () => new Map(unionColumns.map(c => [c.name, c.availableCount])), + [unionColumns], + ); + + const data = useMemo( + () => + unionColumns.map(c => ({ + value: c.name, + label: c.name, + })), + [unionColumns], + ); + + const renderOption = useCallback( + ({ option }: { option: { value: string; label: string } }) => { + const available = availabilityByName.get(option.value) ?? 0; + return ( + + + {option.label} + + {available < totalSources && ( + + {available}/{totalSources} sources + + )} + + ); + }, + [availabilityByName, totalSources], + ); + + return ( + + ); +} diff --git a/packages/app/src/components/MultiSourceRowTable.tsx b/packages/app/src/components/MultiSourceRowTable.tsx new file mode 100644 index 0000000000..20c35784b5 --- /dev/null +++ b/packages/app/src/components/MultiSourceRowTable.tsx @@ -0,0 +1,472 @@ +import { useCallback, useMemo, useState } from 'react'; +import { useQueryState } from 'nuqs'; +import { + chSqlToAliasMap, + ClickHouseQueryError, + convertCHDataTypeToJSType, + isJSDataTypeJSONStringifiable, + JSDataType, +} from '@hyperdx/common-utils/dist/clickhouse'; +import { MULTI_SOURCE_ALIASES } from '@hyperdx/common-utils/dist/core/searchChartConfig'; +import { + BuilderChartConfigWithDateRange, + SourceKind, + TSource, +} from '@hyperdx/common-utils/dist/types'; +import { Flex, Group, Loader, Text, Tooltip } from '@mantine/core'; +import { IconAlertTriangle } from '@tabler/icons-react'; + +import api from '@/api'; +import { searchChartConfigDefaults } from '@/defaults'; +import useOffsetPaginatedQuery from '@/hooks/useOffsetPaginatedQuery'; +import useRowWhere, { RowWhereResult, WithClause } from '@/hooks/useRowWhere'; +import { + mergeStreams, + MULTI_SOURCE_ROW_FIELDS, + StreamSnapshot, +} from '@/utils/multiSourceMerge'; +import { parseAsStringEncoded } from '@/utils/queryParsers'; + +import ChartErrorState from './charts/ChartErrorState'; +import DBRowSidePanel, { + RowSidePanelContext, + RowSidePanelContextProps, +} from './DBRowSidePanel'; +import { RawLogTable, useConfigWithAdditionalSelect } from './DBRowTable'; +import { RowOverviewPanelWrapper } from './DBSqlRowTableWithSidebar'; +import { getMultiSourceColor, SourceBadge } from './MultiSourceBadge'; + +/** One selected source plus its fully-built (canonical-SELECT) chart config. */ +export type MultiSourceStreamSpec = { + source: TSource; + config: BuilderChartConfigWithDateRange; +}; + +// Placeholder config for unused hook slots. The metadata hooks inside +// useConfigWithAdditionalSelect self-disable on empty table names, and the +// paginated query slot is explicitly disabled, so this never reaches +// ClickHouse. +const STUB_CONFIG: BuilderChartConfigWithDateRange = { + connection: '', + from: { databaseName: '', tableName: '' }, + timestampValueExpression: '', + select: '', + where: '', + whereLanguage: 'sql', + dateRange: [new Date(0), new Date(0)], +}; + +const EMPTY_CHSQL = { sql: '', params: {} }; +const EMPTY_EXTRA_COLUMNS: string[] = []; + +type SourceStream = { + spec: MultiSourceStreamSpec | undefined; + data: ReturnType['data']; + fetchNextPage: ReturnType['fetchNextPage']; + hasNextPage: boolean; + isFetching: boolean; + isError: boolean; + error: Error | ClickHouseQueryError | null; + getRowWhere: (row: Record) => RowWhereResult; +}; + +/** + * One source's independent query pipeline: the same + * defaults → additional-key SELECT merge → windowed offset pagination → + * row-WHERE machinery as the single-source DBSqlRowTable, packaged per slot. + * + * Always called (fixed hook count — see MAX_SEARCH_SOURCES); unused slots get + * a stub config and stay disabled. + */ +function useSourceStream( + spec: MultiSourceStreamSpec | undefined, + { + enabled, + isLive, + enableSmallFirstWindow, + queryKeyPrefix, + }: { + enabled: boolean; + isLive: boolean; + enableSmallFirstWindow?: boolean; + queryKeyPrefix?: string; + }, +): SourceStream { + const { data: me } = api.useMe(); + + const configWithDefaults = useMemo( + () => ({ + ...searchChartConfigDefaults(me?.team), + ...(spec?.config ?? STUB_CONFIG), + }), + [me, spec?.config], + ); + + const mergedConfig = useConfigWithAdditionalSelect( + configWithDefaults, + spec?.source.id, + ); + + const { data, fetchNextPage, hasNextPage, isFetching, isError, error } = + useOffsetPaginatedQuery(mergedConfig ?? configWithDefaults, { + enabled: enabled && spec != null && mergedConfig != null, + isLive, + queryKeyPrefix, + enableSmallFirstWindow, + }); + + const aliasMap = useMemo(() => { + const map = chSqlToAliasMap(data?.chSql ?? EMPTY_CHSQL); + // NULL-literal projections (`NULL AS "__hdx_duration_ms"` where a source + // lacks the field) are dropped by the SQL alias parser. Backfill them so + // the row-WHERE clause emits `isNull(NULL)` rather than referencing the + // alias as a (nonexistent) table column. ClickHouse reports NULL literals + // as Nullable(Nothing). + for (const col of data?.meta ?? []) { + if (map[col.name] == null && col.type === 'Nullable(Nothing)') { + map[col.name] = 'NULL'; + } + } + return map; + }, [data]); + + const getRowWhere = useRowWhere({ + meta: data?.meta, + aliasMap, + primaryKeyColumns: mergedConfig?.rowKeyColumns, + }); + + // Stable identity per content change, so downstream merge memos don't + // recompute (and re-sort every fetched row) on unrelated parent renders. + return useMemo( + () => ({ + spec, + data, + fetchNextPage, + hasNextPage: hasNextPage ?? false, + isFetching, + isError, + error: error ?? null, + getRowWhere, + }), + [ + spec, + data, + fetchNextPage, + hasNextPage, + isFetching, + isError, + error, + getRowWhere, + ], + ); +} + +const COLUMN_NAME_MAP: Record = { + [MULTI_SOURCE_ALIASES.timestamp]: 'Timestamp', + [MULTI_SOURCE_ROW_FIELDS.SOURCE_NAME]: 'Source', + [MULTI_SOURCE_ALIASES.service]: 'Service', + [MULTI_SOURCE_ALIASES.severity]: 'Level', + [MULTI_SOURCE_ALIASES.durationMs]: 'Duration (ms)', + [MULTI_SOURCE_ALIASES.body]: 'Message', +}; + +function StreamStatusChips({ streams }: { streams: SourceStream[] }) { + return ( + + {streams.map((stream, i) => { + if (stream.spec == null) return null; + const name = stream.spec.source.name; + return ( + + + {stream.isFetching && } + {stream.isError && ( + + + + + + )} + + ); + })} + + ); +} + +export default function MultiSourceRowTableWithSidebar({ + streams: specs, + isLive, + enabled = true, + extraColumnNames = EMPTY_EXTRA_COLUMNS, + onScroll, + onSidebarOpen, + onExpandedRowsChange, + collapseAllRows, + enableSmallFirstWindow, + tableId, + context, + keepOpenSelector, + queryKeyPrefix, +}: { + /** 2..MAX_SEARCH_SOURCES selected sources with their built configs. */ + streams: MultiSourceStreamSpec[]; + isLive: boolean; + enabled?: boolean; + /** User-picked extra columns projected into every source's SELECT. */ + extraColumnNames?: string[]; + onScroll?: (scrollTop: number) => void; + onSidebarOpen?: (rowId: string) => void; + onExpandedRowsChange?: (hasExpandedRows: boolean) => void; + collapseAllRows?: boolean; + enableSmallFirstWindow?: boolean; + tableId?: string; + context?: RowSidePanelContextProps; + keepOpenSelector?: string; + queryKeyPrefix?: string; +}) { + const streamOpts = { + enabled, + isLive, + enableSmallFirstWindow, + queryKeyPrefix, + }; + + // Fixed hook slots (MAX_SEARCH_SOURCES = 5): hook count stays constant no + // matter how many sources are selected, so no rules-of-hooks gymnastics. + const slot0 = useSourceStream(specs[0], streamOpts); + const slot1 = useSourceStream(specs[1], streamOpts); + const slot2 = useSourceStream(specs[2], streamOpts); + const slot3 = useSourceStream(specs[3], streamOpts); + const slot4 = useSourceStream(specs[4], streamOpts); + + const streams = useMemo( + () => + [slot0, slot1, slot2, slot3, slot4].filter( + (s): s is SourceStream & { spec: MultiSourceStreamSpec } => + s.spec != null, + ), + [slot0, slot1, slot2, slot3, slot4], + ); + + const snapshots: StreamSnapshot[] = useMemo( + () => + streams.map((stream, i) => ({ + sourceId: stream.spec.source.id, + sourceName: stream.spec.source.name, + sourceColor: getMultiSourceColor(i), + rows: stream.data?.data ?? [], + window: stream.data?.window ?? null, + lastPageRowCount: stream.data?.lastPageRowCount ?? null, + hasNextPage: stream.hasNextPage, + isActive: !stream.isError, + dateRange: stream.spec.config.dateRange, + })), + [streams], + ); + + const merged = useMemo( + () => mergeStreams(snapshots, 'DESC', MULTI_SOURCE_ALIASES.timestamp), + [snapshots], + ); + + // Merge column meta across streams by canonical alias name, preferring a + // resolved type over the Nullable(Nothing) a `NULL AS "alias"` projection + // reports. + const columnTypeMap = useMemo(() => { + const map = new Map(); + for (const stream of streams) { + for (const col of stream.data?.meta ?? []) { + const jsType = convertCHDataTypeToJSType(col.type); + const existing = map.get(col.name); + if (existing == null || existing._type == null) { + map.set(col.name, { _type: jsType }); + } + } + } + map.set(MULTI_SOURCE_ROW_FIELDS.SOURCE_NAME, { + _type: JSDataType.String, + }); + return map; + }, [streams]); + + const includeDuration = specs.some(s => s.source.kind === SourceKind.Trace); + + const displayedColumns = useMemo( + () => [ + MULTI_SOURCE_ALIASES.timestamp, + MULTI_SOURCE_ROW_FIELDS.SOURCE_NAME, + MULTI_SOURCE_ALIASES.service, + MULTI_SOURCE_ALIASES.severity, + ...(includeDuration ? [MULTI_SOURCE_ALIASES.durationMs] : []), + ...extraColumnNames, + MULTI_SOURCE_ALIASES.body, + ], + [includeDuration, extraColumnNames], + ); + + // Stringify object-typed cells (Map/Array/JSON) the same way DBSqlRowTable + // does — both for display and because useRowWhere expects the stringified + // form when rebuilding a row WHERE clause. + const rows = useMemo(() => { + const objectColumns = [...columnTypeMap.entries()] + .filter(([, v]) => isJSDataTypeJSONStringifiable(v._type)) + .map(([name]) => name); + if (objectColumns.length === 0) { + return merged.rows; + } + return merged.rows.map(row => { + const newRow = { ...row }; + for (const col of objectColumns) { + if (!(col in newRow) || newRow[col] == null) continue; + if (columnTypeMap.get(col)?._type === JSDataType.JSON) { + newRow[col] = JSON.stringify(newRow[col]).replace(/\//g, '\\/'); + } else { + newRow[col] = JSON.stringify(newRow[col]); + } + } + return newRow; + }); + }, [merged.rows, columnTypeMap]); + + // Row identity dispatches to the row's own stream: each stream has its own + // result meta / alias map / primary-key columns. The client-side tag fields + // are stripped first — they aren't real columns. + const generateRowId = useCallback( + (row: Record): RowWhereResult => { + const { + [MULTI_SOURCE_ROW_FIELDS.SOURCE_ID]: sourceId, + [MULTI_SOURCE_ROW_FIELDS.SOURCE_NAME]: _name, + [MULTI_SOURCE_ROW_FIELDS.SOURCE_COLOR]: _color, + ...dbRow + } = row; + const stream = streams.find(s => s.spec.source.id === sourceId); + if (stream == null) { + return { where: '', aliasWith: [] }; + } + return stream.getRowWhere(dbRow); + }, + [streams], + ); + + // Advance only the stream(s) holding the frontier back; the leaders keep + // their fetched-but-held rows until the laggards catch up. + const fetchNextPage = useCallback(() => { + for (const sourceId of merged.laggingSourceIds) { + const stream = streams.find(s => s.spec.source.id === sourceId); + stream?.fetchNextPage({ cancelRefetch: false }); + } + }, [merged.laggingSourceIds, streams]); + + const hasNextPage = streams.some(s => !s.isError && s.hasNextPage); + const isFetching = streams.some(s => s.isFetching); + const allFailed = streams.length > 0 && streams.every(s => s.isError); + const firstError = streams.find(s => s.error != null)?.error ?? undefined; + + // Side panel wiring — the same URL-param contract as + // DBSqlRowTableWithSideBar, except the panel's source comes from the + // clicked row rather than the (single) searched source. + const [rowId, setRowId] = useQueryState('rowWhere', parseAsStringEncoded); + const [rowSource, setRowSource] = useQueryState('rowSource'); + const [aliasWith, setAliasWith] = useState([]); + + const onRowDetailsClick = useCallback( + (row: Record) => { + const rowWhere = generateRowId(row); + if (!rowWhere.where) return; + setRowId(rowWhere.where); + setAliasWith(rowWhere.aliasWith); + setRowSource(row[MULTI_SOURCE_ROW_FIELDS.SOURCE_ID] ?? null); + onSidebarOpen?.(rowWhere.where); + }, + [generateRowId, setRowId, setRowSource, onSidebarOpen], + ); + + const onCloseSidebar = useCallback(() => { + setRowId(null); + setRowSource(null); + }, [setRowId, setRowSource]); + + const panelSource = useMemo( + () => specs.find(s => s.source.id === rowSource)?.source, + [specs, rowSource], + ); + + const renderRowDetails = useCallback( + (r: { id: string; aliasWith?: WithClause[]; [key: string]: unknown }) => { + const source = specs.find( + s => s.source.id === r[MULTI_SOURCE_ROW_FIELDS.SOURCE_ID], + )?.source; + if (!source) { + return
Loading...
; + } + return ( + + ); + }, + [specs], + ); + + const loadingDate = + merged.frontier != null && hasNextPage + ? new Date(merged.frontier) + : undefined; + + const firstConfig = streams[0]?.spec.config; + + return ( + + {panelSource != null && ( + + )} + + + {allFailed ? ( + + ) : ( + + )} + + + ); +} diff --git a/packages/app/src/components/MultiSourceTimeChart.tsx b/packages/app/src/components/MultiSourceTimeChart.tsx new file mode 100644 index 0000000000..afb8dbdfba --- /dev/null +++ b/packages/app/src/components/MultiSourceTimeChart.tsx @@ -0,0 +1,320 @@ +import { useMemo, useState } from 'react'; +import { + ColumnMetaType, + filterColumnMetaByType, + JSDataType, + ResponseJSON, +} from '@hyperdx/common-utils/dist/clickhouse'; +import { + BuilderChartConfigWithDateRange, + DisplayType, + TSource, +} from '@hyperdx/common-utils/dist/types'; +import { Text } from '@mantine/core'; +import { keepPreviousData } from '@tanstack/react-query'; + +import api from '@/api'; +import { + convertToTimeChartConfig, + formatResponseForTimeChart, + useTimeChartSettings, +} from '@/ChartUtils'; +import ChartContainer from '@/components/charts/ChartContainer'; +import ChartErrorState from '@/components/charts/ChartErrorState'; +import { type ActiveClickPayload, MemoChart } from '@/HDXMultiSeriesTimeChart'; +import { useQueriedChartConfig } from '@/hooks/useChartConfig'; +import type { NumberFormat } from '@/types'; + +import { getMultiSourceColor } from './MultiSourceBadge'; + +/** Synthetic group column tagged onto each source's histogram rows. */ +const SOURCE_GROUP_COLUMN = '__hdx_source'; + +export type MultiSourceChartSpec = { + source: TSource; + /** Per-source count() histogram config (canonical WHERE, no groupBy). */ + config: BuilderChartConfigWithDateRange; +}; + +// Placeholder for unused hook slots; never queried (enabled: false). +const STUB_CONFIG: BuilderChartConfigWithDateRange = { + connection: '', + from: { databaseName: '', tableName: '' }, + timestampValueExpression: '', + select: '', + where: '', + whereLanguage: 'sql', + dateRange: [new Date(0), new Date(0)], +}; + +function useHistogramSlot( + spec: MultiSourceChartSpec | undefined, + { + enabled, + queryKeyPrefix, + enableParallelQueries, + parallelizeWhenPossible, + }: { + enabled: boolean; + queryKeyPrefix: string; + enableParallelQueries?: boolean; + parallelizeWhenPossible?: boolean; + }, +) { + const queriedConfig = useMemo( + () => convertToTimeChartConfig(spec?.config ?? STUB_CONFIG), + [spec?.config], + ); + + return useQueriedChartConfig(queriedConfig, { + // Key shape mirrors DBTimeChart/SearchTotalCountChart so TanStack can + // de-dupe the histogram and total-count consumers of the same source. + queryKey: [ + queryKeyPrefix, + queriedConfig, + 'chunked', + { + disableQueryChunking: false, + enableParallelQueries, + parallelizeWhenPossible, + }, + ], + placeholderData: keepPreviousData, + enableQueryChunking: true, + enableParallelQueries: enableParallelQueries && parallelizeWhenPossible, + enabled: enabled && spec != null, + }); +} + +/** + * Runs one count() histogram query per selected source (fixed hook slots, see + * MAX_SEARCH_SOURCES) and merges the responses into a single response shape + * with a synthetic source-name group column — so the standard time-chart + * transform naturally yields one series per source. + */ +function useMultiSourceHistogram( + specs: MultiSourceChartSpec[], + { + enabled = true, + queryKeyPrefix, + enableParallelQueries, + }: { + enabled?: boolean; + queryKeyPrefix: string; + enableParallelQueries?: boolean; + }, +) { + const { data: me, isLoading: isLoadingMe } = api.useMe(); + const slotOpts = { + enabled: enabled && !isLoadingMe, + queryKeyPrefix, + enableParallelQueries, + parallelizeWhenPossible: me?.team?.parallelizeWhenPossible, + }; + + const slot0 = useHistogramSlot(specs[0], slotOpts); + const slot1 = useHistogramSlot(specs[1], slotOpts); + const slot2 = useHistogramSlot(specs[2], slotOpts); + const slot3 = useHistogramSlot(specs[3], slotOpts); + const slot4 = useHistogramSlot(specs[4], slotOpts); + + const slots = [slot0, slot1, slot2, slot3, slot4].slice(0, specs.length); + + const isLoading = slots.some(s => s.isLoading); + const allFailed = slots.length > 0 && slots.every(s => s.isError); + const anyError = slots.some(s => s.isError); + const error = slots.find(s => s.error != null)?.error ?? undefined; + const isComplete = + slots.length > 0 && slots.every(s => s.isError || !!s.data?.isComplete); + + const mergedResponse: ResponseJSON> | undefined = + useMemo(() => { + const slotData = [ + slot0.data, + slot1.data, + slot2.data, + slot3.data, + slot4.data, + ]; + let meta: ColumnMetaType[] | undefined; + const data: Record[] = []; + for (let i = 0; i < specs.length; i++) { + const response = slotData[i]; + if (response?.meta == null || response.meta.length === 0) continue; + if (meta == null) { + meta = [ + ...response.meta, + { name: SOURCE_GROUP_COLUMN, type: 'String' }, + ]; + } + const sourceName = specs[i].source.name; + for (const row of response.data ?? []) { + data.push({ ...row, [SOURCE_GROUP_COLUMN]: sourceName }); + } + } + return meta ? { data, meta, rows: data.length } : undefined; + }, [slot0.data, slot1.data, slot2.data, slot3.data, slot4.data, specs]); + + return { mergedResponse, isLoading, allFailed, anyError, error, isComplete }; +} + +const EMPTY_NUMBER_FORMATS = new Map(); + +/** + * The multi-source search histogram: one stacked count() series per selected + * source, colored consistently with the results-table badges. A thin + * counterpart to DBTimeChart — drag-to-zoom and the legend work; per-series + * drill-down/pinned tooltips are single-source features and are omitted. + */ +export function MultiSourceTimeChart({ + specs, + enabled = true, + queryKeyPrefix, + enableParallelQueries, + onTimeRangeSelect, + showLegend = true, +}: { + specs: MultiSourceChartSpec[]; + enabled?: boolean; + queryKeyPrefix: string; + enableParallelQueries?: boolean; + onTimeRangeSelect?: (start: Date, end: Date) => void; + showLegend?: boolean; +}) { + const { mergedResponse, isLoading, allFailed, error, isComplete } = + useMultiSourceHistogram(specs, { + enabled, + queryKeyPrefix, + enableParallelQueries, + }); + + const firstConfig = specs[0]?.config; + const { dateRange, granularity } = useTimeChartSettings( + firstConfig ?? STUB_CONFIG, + ); + + const [activeClickPayload, setActiveClickPayload] = useState< + ActiveClickPayload | undefined + >(); + + const colorBySourceName = useMemo( + () => + new Map( + specs.map((spec, i) => [spec.source.name, getMultiSourceColor(i)]), + ), + [specs], + ); + + const formatted = useMemo(() => { + if (mergedResponse == null) { + return null; + } + try { + const result = formatResponseForTimeChart({ + currentPeriodResponse: mergedResponse, + dateRange, + granularity, + generateEmptyBuckets: true, + }); + // One series per source: recolor to the shared per-source palette so + // the histogram matches the table badges and status chips. + for (const line of result.lineData) { + const color = colorBySourceName.get(line.dataKey); + if (color != null) { + line.color = color; + } + } + return result; + } catch (e) { + console.error(e); + return null; + } + }, [mergedResponse, dateRange, granularity, colorBySourceName]); + + if (allFailed && error) { + return ; + } + + return ( + + {isLoading && formatted == null ? ( +
+ Loading Chart Data... +
+ ) : formatted == null || formatted.graphResults.length === 0 ? ( +
+ No data found within time range. +
+ ) : ( + + )} + + ); +} + +/** + * Summed "N Results" across every selected source, sharing the histogram's + * per-source queries (identical query keys) so it adds no ClickHouse load. + */ +export function MultiSourceTotalCountChart({ + specs, + enabled = true, + queryKeyPrefix, + enableParallelQueries, +}: { + specs: MultiSourceChartSpec[]; + enabled?: boolean; + queryKeyPrefix: string; + enableParallelQueries?: boolean; +}) { + const { mergedResponse, isLoading, allFailed } = useMultiSourceHistogram( + specs, + { + enabled, + queryKeyPrefix, + enableParallelQueries, + }, + ); + + const totalCount = useMemo(() => { + if (mergedResponse == null) return undefined; + // The count column may be renamed (e.g. via materialized views); fall back + // to the first numeric column, mirroring SearchTotalCountChart. + const countColumn = + mergedResponse.meta?.find(c => c.name === 'count()')?.name ?? + filterColumnMetaByType(mergedResponse.meta ?? [], [ + JSDataType.Number, + ])?.[0]?.name ?? + 'count()'; + return mergedResponse.data.reduce( + (sum: number, row: any) => sum + (Number.parseInt(row[countColumn]) || 0), + 0, + ); + }, [mergedResponse]); + + return ( + + {isLoading && totalCount == null ? ( + ··· Results + ) : totalCount != null && !allFailed ? ( + `${totalCount.toLocaleString()} Results` + ) : ( + '0 Results' + )} + + ); +} diff --git a/packages/app/src/defaults.ts b/packages/app/src/defaults.ts index 5894d8b78d..cf850a2fd2 100644 --- a/packages/app/src/defaults.ts +++ b/packages/app/src/defaults.ts @@ -2,6 +2,14 @@ import type { BuilderChartConfigWithDateRange } from '@hyperdx/common-utils/dist // Limit defaults export const DEFAULT_SEARCH_ROW_LIMIT = 200; + +// Ceiling on how many sources one search can span. Each selected source runs +// its own result stream plus histogram/count aggregates, so the fan-out cost +// is N× a single search; 5 keeps the worst case bounded while covering the +// common "a few log sources + traces" setups. Also the size of the fixed +// hook-slot arrays in the multi-source components — raising it means adding +// slots there too. +export const MAX_SEARCH_SOURCES = 5; export const DEFAULT_QUERY_TIMEOUT = 60; // max_execution_time, seconds export const DEFAULT_FILTER_KEYS_FETCH_LIMIT = 100; export const DEFAULT_SERIES_LIMIT = 100; diff --git a/packages/app/src/hooks/useMultiSourceSearch.ts b/packages/app/src/hooks/useMultiSourceSearch.ts new file mode 100644 index 0000000000..9493bbd848 --- /dev/null +++ b/packages/app/src/hooks/useMultiSourceSearch.ts @@ -0,0 +1,91 @@ +import { useMemo } from 'react'; +import { MultiSourceExtraColumn } from '@hyperdx/common-utils/dist/core/searchChartConfig'; +import { TSource } from '@hyperdx/common-utils/dist/types'; + +import { useColumns } from '@/hooks/useMetadata'; + +const EMPTY_SOURCE_PARAMS = { + databaseName: '', + tableName: '', + connectionId: '', +}; + +function columnsParamsFor(source: TSource | undefined) { + if (source == null) return EMPTY_SOURCE_PARAMS; + return { + databaseName: source.from.databaseName, + tableName: source.from.tableName, + connectionId: source.connection, + }; +} + +export type MultiSourceColumnOption = { + name: string; + /** How many of the selected sources have this column. */ + availableCount: number; +}; + +/** + * Top-level columns (DESCRIBE) for each selected source of a multi-source + * search, plus the deduped union with per-column availability counts for the + * add-column picker. Fixed hook slots (MAX_SEARCH_SOURCES = 5); useColumns + * self-disables for empty slots. + */ +export function useMultiSourceColumns(sources: TSource[]): { + columnsBySourceId: Map>; + unionColumns: MultiSourceColumnOption[]; +} { + const q0 = useColumns(columnsParamsFor(sources[0])); + const q1 = useColumns(columnsParamsFor(sources[1])); + const q2 = useColumns(columnsParamsFor(sources[2])); + const q3 = useColumns(columnsParamsFor(sources[3])); + const q4 = useColumns(columnsParamsFor(sources[4])); + + return useMemo(() => { + const slotData = [q0.data, q1.data, q2.data, q3.data, q4.data]; + const columnsBySourceId = new Map>(); + const availability = new Map(); + + for (let i = 0; i < sources.length; i++) { + const source = sources[i]; + const columns = slotData[i]; + if (source == null || columns == null) continue; + const names = new Set(columns.map(c => c.name)); + columnsBySourceId.set(source.id, names); + for (const name of names) { + availability.set(name, (availability.get(name) ?? 0) + 1); + } + } + + const unionColumns = [...availability.entries()] + .map(([name, availableCount]) => ({ name, availableCount })) + .sort( + (a, b) => + b.availableCount - a.availableCount || a.name.localeCompare(b.name), + ); + + return { columnsBySourceId, unionColumns }; + }, [q0.data, q1.data, q2.data, q3.data, q4.data, sources]); +} + +/** Quote a column name as a ClickHouse identifier when it needs it. */ +function quoteIdentifier(name: string): string { + return /^[A-Za-z_][A-Za-z0-9_]*$/.test(name) + ? name + : `\`${name.replace(/`/g, '\\`')}\``; +} + +/** + * Resolve the user-picked extra column names into per-source SELECT + * expressions: the (quoted) column itself where the source's table has it, + * NULL otherwise — so every source still returns the same result shape. + */ +export function resolveExtraColumnsForSource( + extraColumnNames: string[], + sourceColumns: Set | undefined, +): MultiSourceExtraColumn[] { + return extraColumnNames.map(name => ({ + name, + expression: sourceColumns?.has(name) ? quoteIdentifier(name) : null, + })); +} diff --git a/packages/app/src/hooks/useOffsetPaginatedQuery.tsx b/packages/app/src/hooks/useOffsetPaginatedQuery.tsx index 5a27d39b96..2dabb32720 100644 --- a/packages/app/src/hooks/useOffsetPaginatedQuery.tsx +++ b/packages/app/src/hooks/useOffsetPaginatedQuery.tsx @@ -427,7 +427,9 @@ function flattenPages(pages: TQueryFnData[]) { return pages.flatMap(p => p.data); } -function flattenData(data: TData | undefined): TQueryFnData | null { +function flattenData( + data: TData | undefined, +): (TQueryFnData & { lastPageRowCount: number }) | null { if (data == null || data.pages.length === 0) { return null; } @@ -437,6 +439,10 @@ function flattenData(data: TData | undefined): TQueryFnData | null { data: flattenPages(data.pages), chSql: data.pages[0].chSql, window: data.pages[data.pages.length - 1].window, + // Whether the last fetched page hit results distinguishes "still mid-window + // at LIMIT" from "window drained" — multi-source merge uses this to compute + // how far this stream's time coverage safely extends. + lastPageRowCount: data.pages[data.pages.length - 1].data.length, }; } diff --git a/packages/app/src/hooks/useResolvedSourcesParam.ts b/packages/app/src/hooks/useResolvedSourcesParam.ts new file mode 100644 index 0000000000..deb2f449d1 --- /dev/null +++ b/packages/app/src/hooks/useResolvedSourcesParam.ts @@ -0,0 +1,58 @@ +import { useEffect, useMemo } from 'react'; +import { SourceKind, TSource } from '@hyperdx/common-utils/dist/types'; +import { notifications } from '@mantine/notifications'; + +import { MAX_SEARCH_SOURCES } from '@/defaults'; +import { useSources } from '@/source'; +import { resolveSourcesParam } from '@/utils/sourceParams'; + +const EMPTY_SOURCES: TSource[] = []; + +/** + * Resolves the multi-source search param (a list of source IDs or names) to + * the matching sources, deduped and capped at MAX_SEARCH_SOURCES. + * + * Elements that don't match any usable source are dropped from the selection + * and reported once via a Mantine warning, mirroring useResolvedSourceParam. + */ +export function useResolvedSourcesParam( + paramValues: string[] | null | undefined, + { kinds }: { kinds?: SourceKind[] } = {}, +): { sources: TSource[] } { + const { data: allSources } = useSources(); + + // Key the memo on a serialized `kinds` so callers can pass inline arrays + // without breaking memoization. + const kindsKey = kinds?.join(','); + const { sources, unresolvedKey } = useMemo(() => { + const allKinds = new Set(Object.values(SourceKind)); + const resolvedKinds = kindsKey + ? kindsKey.split(',').filter((k): k is SourceKind => allKinds.has(k)) + : undefined; + const resolution = resolveSourcesParam(paramValues, allSources, { + kinds: resolvedKinds, + max: MAX_SEARCH_SOURCES, + }); + if (resolution.status !== 'resolved') { + return { sources: EMPTY_SOURCES, unresolvedKey: undefined }; + } + return { + sources: resolution.sources.length ? resolution.sources : EMPTY_SOURCES, + unresolvedKey: resolution.unresolved.length + ? resolution.unresolved.join(', ') + : undefined, + }; + }, [paramValues, allSources, kindsKey]); + + useEffect(() => { + if (unresolvedKey == null) return; + notifications.show({ + id: 'sources-param-unresolved-' + unresolvedKey, + color: 'yellow', + title: 'Some sources were not found', + message: `No searchable source matches: ${unresolvedKey}. They may have been renamed or deleted.`, + }); + }, [unresolvedKey]); + + return useMemo(() => ({ sources }), [sources]); +} diff --git a/packages/app/src/utils/__tests__/multiSourceMerge.test.ts b/packages/app/src/utils/__tests__/multiSourceMerge.test.ts new file mode 100644 index 0000000000..ae77b18f9e --- /dev/null +++ b/packages/app/src/utils/__tests__/multiSourceMerge.test.ts @@ -0,0 +1,379 @@ +import { + computeFrontier, + coveredUntil, + mergeStreams, + MULTI_SOURCE_ROW_FIELDS, + StreamSnapshot, +} from '@/utils/multiSourceMerge'; + +const TS_KEY = '__hdx_timestamp'; + +const T = (iso: string) => new Date(iso); +const ms = (iso: string) => new Date(iso).getTime(); + +// Search range: 10:00 - 12:00 UTC +const DATE_RANGE: [Date, Date] = [ + T('2026-08-07T10:00:00Z'), + T('2026-08-07T12:00:00Z'), +]; + +const row = (iso: string, extra: Record = {}) => ({ + [TS_KEY]: iso, + ...extra, +}); + +const makeStream = ( + overrides: Partial & { sourceId: string }, +): StreamSnapshot => ({ + sourceName: overrides.sourceId, + rows: [], + window: null, + lastPageRowCount: null, + hasNextPage: true, + isActive: true, + dateRange: DATE_RANGE, + ...overrides, +}); + +const parseTs = (r: Record) => new Date(r[TS_KEY]).getTime(); + +describe('coveredUntil (DESC)', () => { + it('covers the whole range when the stream is fully drained', () => { + const stream = makeStream({ + sourceId: 'a', + hasNextPage: false, + window: { + startTime: T('2026-08-07T10:00:00Z'), + endTime: T('2026-08-07T12:00:00Z'), + }, + lastPageRowCount: 0, + }); + expect(coveredUntil(stream, 'DESC', parseTs)).toBe(DATE_RANGE[0].getTime()); + }); + + it('covers nothing during the initial fetch even though hasNextPage is still false', () => { + // useInfiniteQuery reports hasNextPage=false before the first page lands; + // that must not be mistaken for a drained stream. + const stream = makeStream({ sourceId: 'a', hasNextPage: false }); + expect(coveredUntil(stream, 'DESC', parseTs)).toBe(DATE_RANGE[1].getTime()); + }); + + it('covers nothing before the first page arrives', () => { + const stream = makeStream({ sourceId: 'a' }); + expect(coveredUntil(stream, 'DESC', parseTs)).toBe(DATE_RANGE[1].getTime()); + }); + + it('covers through the window start when the last page was empty', () => { + const stream = makeStream({ + sourceId: 'a', + window: { + startTime: T('2026-08-07T11:45:00Z'), + endTime: T('2026-08-07T12:00:00Z'), + }, + lastPageRowCount: 0, + }); + expect(coveredUntil(stream, 'DESC', parseTs)).toBe( + ms('2026-08-07T11:45:00Z'), + ); + }); + + it('covers only through the oldest fetched row when stopped mid-window at LIMIT', () => { + const stream = makeStream({ + sourceId: 'a', + rows: [row('2026-08-07T11:59:00Z'), row('2026-08-07T11:50:00Z')], + window: { + startTime: T('2026-08-07T11:45:00Z'), + endTime: T('2026-08-07T12:00:00Z'), + }, + lastPageRowCount: 2, + }); + expect(coveredUntil(stream, 'DESC', parseTs)).toBe( + ms('2026-08-07T11:50:00Z'), + ); + }); +}); + +describe('coveredUntil (ASC)', () => { + it('mirrors the DESC semantics from the start of the range', () => { + expect(coveredUntil(makeStream({ sourceId: 'a' }), 'ASC', parseTs)).toBe( + DATE_RANGE[0].getTime(), + ); + expect( + coveredUntil( + makeStream({ + sourceId: 'a', + hasNextPage: false, + window: { + startTime: T('2026-08-07T10:00:00Z'), + endTime: T('2026-08-07T12:00:00Z'), + }, + lastPageRowCount: 0, + }), + 'ASC', + parseTs, + ), + ).toBe(DATE_RANGE[1].getTime()); + expect( + coveredUntil( + makeStream({ + sourceId: 'a', + window: { + startTime: T('2026-08-07T10:00:00Z'), + endTime: T('2026-08-07T10:15:00Z'), + }, + lastPageRowCount: 0, + }), + 'ASC', + parseTs, + ), + ).toBe(ms('2026-08-07T10:15:00Z')); + }); +}); + +const drainedStream = (sourceId: string) => + makeStream({ + sourceId, + hasNextPage: false, + window: { + startTime: T('2026-08-07T10:00:00Z'), + endTime: T('2026-08-07T12:00:00Z'), + }, + lastPageRowCount: 0, + }); + +describe('computeFrontier', () => { + it('is the least-covered active stream (max for DESC)', () => { + const drained = drainedStream('a'); + const midWindow = makeStream({ + sourceId: 'b', + rows: [row('2026-08-07T11:50:00Z')], + window: { + startTime: T('2026-08-07T11:45:00Z'), + endTime: T('2026-08-07T12:00:00Z'), + }, + lastPageRowCount: 1, + }); + expect(computeFrontier([drained, midWindow], 'DESC', parseTs)).toBe( + ms('2026-08-07T11:50:00Z'), + ); + }); + + it('ignores inactive (errored/excluded) streams so they cannot stall the merge', () => { + const drained = drainedStream('a'); + const errored = makeStream({ sourceId: 'b', isActive: false }); + expect(computeFrontier([drained, errored], 'DESC', parseTs)).toBe( + DATE_RANGE[0].getTime(), + ); + }); + + it('is null when no stream is active', () => { + const errored = makeStream({ sourceId: 'a', isActive: false }); + expect(computeFrontier([errored], 'DESC', parseTs)).toBeNull(); + }); +}); + +describe('mergeStreams', () => { + const window0 = { + startTime: T('2026-08-07T11:45:00Z'), + endTime: T('2026-08-07T12:00:00Z'), + }; + + it('interleaves rows across streams newest-first and tags their source', () => { + const a = makeStream({ + sourceId: 'a', + sourceName: 'app logs', + rows: [row('2026-08-07T11:59:00Z'), row('2026-08-07T11:57:00Z')], + window: window0, + lastPageRowCount: 2, + hasNextPage: false, + }); + const b = makeStream({ + sourceId: 'b', + sourceName: 'traces', + rows: [row('2026-08-07T11:58:00Z')], + window: window0, + lastPageRowCount: 1, + hasNextPage: false, + }); + + const { rows } = mergeStreams([a, b], 'DESC', TS_KEY); + + expect(rows.map(r => r[TS_KEY])).toEqual([ + '2026-08-07T11:59:00Z', + '2026-08-07T11:58:00Z', + '2026-08-07T11:57:00Z', + ]); + expect(rows.map(r => r[MULTI_SOURCE_ROW_FIELDS.SOURCE_NAME])).toEqual([ + 'app logs', + 'traces', + 'app logs', + ]); + expect(rows[0][MULTI_SOURCE_ROW_FIELDS.SOURCE_ID]).toBe('a'); + }); + + it('holds back rows older than the frontier until lagging streams catch up', () => { + // Stream a is fully drained down to 10:00; stream b stopped at LIMIT with + // its oldest row at 11:50 — anything older than 11:50 from a must wait. + const a = makeStream({ + sourceId: 'a', + rows: [ + row('2026-08-07T11:55:00Z'), + row('2026-08-07T11:49:00Z'), // older than b's coverage — held back + ], + window: window0, + lastPageRowCount: 2, + hasNextPage: false, + }); + const b = makeStream({ + sourceId: 'b', + rows: [row('2026-08-07T11:50:00Z')], + window: window0, + lastPageRowCount: 1, + hasNextPage: true, + }); + + const { rows, frontier, laggingSourceIds } = mergeStreams( + [a, b], + 'DESC', + TS_KEY, + ); + + expect(frontier).toBe(ms('2026-08-07T11:50:00Z')); + expect(rows.map(r => r[TS_KEY])).toEqual([ + '2026-08-07T11:55:00Z', + '2026-08-07T11:50:00Z', + ]); + expect(laggingSourceIds).toEqual(['b']); + }); + + it('shows everything when all streams are drained', () => { + const a = makeStream({ + sourceId: 'a', + rows: [row('2026-08-07T10:05:00Z')], + window: window0, + lastPageRowCount: 1, + hasNextPage: false, + }); + const b = makeStream({ + sourceId: 'b', + rows: [row('2026-08-07T10:03:00Z')], + window: window0, + lastPageRowCount: 1, + hasNextPage: false, + }); + + const { rows, laggingSourceIds } = mergeStreams([a, b], 'DESC', TS_KEY); + + expect(rows).toHaveLength(2); + expect(laggingSourceIds).toEqual([]); + }); + + it('holds everything back while a stream has no page yet, without marking it lagging', () => { + const a = makeStream({ + sourceId: 'a', + rows: [row('2026-08-07T11:59:00Z')], + window: window0, + lastPageRowCount: 1, + hasNextPage: false, + }); + const pending = makeStream({ sourceId: 'b' }); + + const { rows, laggingSourceIds } = mergeStreams( + [a, pending], + 'DESC', + TS_KEY, + ); + + // Frontier sits at the range end until b's first page lands. + expect(rows).toEqual([]); + // b's initial fetch is already in flight — nothing to advance. + expect(laggingSourceIds).toEqual([]); + }); + + it('still shows rows from errored streams but never waits on them', () => { + const a = makeStream({ + sourceId: 'a', + rows: [row('2026-08-07T11:59:00Z')], + window: window0, + lastPageRowCount: 1, + hasNextPage: false, + }); + const errored = makeStream({ + sourceId: 'b', + rows: [row('2026-08-07T11:58:00Z')], + window: window0, + lastPageRowCount: 1, + isActive: false, + }); + + const { rows, laggingSourceIds } = mergeStreams( + [a, errored], + 'DESC', + TS_KEY, + ); + + expect(rows.map(r => r[TS_KEY])).toEqual([ + '2026-08-07T11:59:00Z', + '2026-08-07T11:58:00Z', + ]); + expect(laggingSourceIds).toEqual([]); + }); + + it('merges oldest-first with a mirrored frontier for ASC', () => { + const window0Asc = { + startTime: T('2026-08-07T10:00:00Z'), + endTime: T('2026-08-07T10:15:00Z'), + }; + const a = makeStream({ + sourceId: 'a', + rows: [ + row('2026-08-07T10:01:00Z'), + row('2026-08-07T10:20:00Z'), // beyond b's coverage — held back + ], + window: window0Asc, + lastPageRowCount: 2, + hasNextPage: false, + }); + const b = makeStream({ + sourceId: 'b', + rows: [row('2026-08-07T10:05:00Z')], + window: window0Asc, + lastPageRowCount: 1, + hasNextPage: true, + }); + + const { rows, frontier, laggingSourceIds } = mergeStreams( + [a, b], + 'ASC', + TS_KEY, + ); + + expect(frontier).toBe(ms('2026-08-07T10:05:00Z')); + expect(rows.map(r => r[TS_KEY])).toEqual([ + '2026-08-07T10:01:00Z', + '2026-08-07T10:05:00Z', + ]); + expect(laggingSourceIds).toEqual(['b']); + }); + + it('advances every stream tied at the frontier', () => { + const a = makeStream({ + sourceId: 'a', + rows: [row('2026-08-07T11:50:00Z')], + window: window0, + lastPageRowCount: 1, + hasNextPage: true, + }); + const b = makeStream({ + sourceId: 'b', + rows: [row('2026-08-07T11:50:00Z')], + window: window0, + lastPageRowCount: 1, + hasNextPage: true, + }); + + const { laggingSourceIds } = mergeStreams([a, b], 'DESC', TS_KEY); + + expect(laggingSourceIds).toEqual(['a', 'b']); + }); +}); diff --git a/packages/app/src/utils/multiSourceMerge.ts b/packages/app/src/utils/multiSourceMerge.ts new file mode 100644 index 0000000000..ae53b730c4 --- /dev/null +++ b/packages/app/src/utils/multiSourceMerge.ts @@ -0,0 +1,239 @@ +/** + * Pure merge logic for multi-source search: k-way merges per-source result + * streams by timestamp, bounded by a "safe frontier" so the interleaved + * timeline never shows a gap another source could still fill. + * + * Every source stream paginates through the same progressive time windows + * (see utils/searchWindows.ts — windows are a pure function of the date + * range), but streams advance at different speeds: one source may be three + * windows deep while another is still mid-window at its row LIMIT. A merged + * DESC timeline is only correct down to the timestamp every stream has + * covered; rows older than that are held back until the lagging streams catch + * up. + */ + +/** Client-side fields tagged onto every merged row. Never sent to ClickHouse. */ +export const MULTI_SOURCE_ROW_FIELDS = { + SOURCE_ID: '__hdx_source_id', + SOURCE_NAME: '__hdx_source_name', + SOURCE_COLOR: '__hdx_source_color', +} as const; + +export type MergeDirection = 'ASC' | 'DESC'; + +export type StreamSnapshot = { + sourceId: string; + sourceName: string; + /** Badge/series color for this source; tagged onto rows for the table cell. */ + sourceColor?: string; + /** + * Rows fetched so far, in stream order (newest-first for DESC, + * oldest-first for ASC) — the order the windowed query produces. + */ + rows: Record[]; + /** The last fetched page's time window; null when no page has completed. */ + window: { startTime: Date; endTime: Date } | null; + /** + * Row count of the last fetched page; 0 means the window was drained, + * >0 means the stream may have stopped mid-window at its LIMIT. + * Null when no page has completed. + */ + lastPageRowCount: number | null; + hasNextPage: boolean; + /** + * Errored/excluded streams don't bound the frontier (they'd stall the merge + * forever); their already-fetched rows are still shown. + */ + isActive: boolean; + /** The full searched range, used when a stream is fully drained. */ + dateRange: [Date, Date]; +}; + +/** + * Epoch-ms timestamp T such that this stream is guaranteed to have produced + * every row it has on the already-covered side of T: + * DESC — all of the stream's rows with ts >= T are fetched; + * ASC — all of the stream's rows with ts <= T are fetched. + * + * Conservative by construction: when the stream stopped mid-window at its + * LIMIT, coverage only extends to the last row it returned, not the window + * boundary. + */ +export function coveredUntil( + stream: StreamSnapshot, + direction: MergeDirection, + parseTs: (row: Record) => number, +): number { + const [start, end] = stream.dateRange; + + if (stream.window == null || stream.lastPageRowCount == null) { + // Nothing fetched yet: no coverage at all. Checked before hasNextPage — + // useInfiniteQuery reports hasNextPage=false during the initial fetch, + // which must not read as "fully drained". + return direction === 'DESC' ? end.getTime() : start.getTime(); + } + + if (!stream.hasNextPage) { + // Fully drained: the stream covered the entire searched range. + return direction === 'DESC' ? start.getTime() : end.getTime(); + } + + if (stream.lastPageRowCount === 0) { + // The last window came back empty, so it is fully covered. + return direction === 'DESC' + ? stream.window.startTime.getTime() + : stream.window.endTime.getTime(); + } + + // Mid-window at LIMIT: covered only through the last row returned. Rows are + // in stream order, so the last row is the furthest-along one. + const lastRow = stream.rows[stream.rows.length - 1]; + if (lastRow == null) { + // Defensive: a non-zero lastPageRowCount implies rows exist. + return direction === 'DESC' + ? stream.window.endTime.getTime() + : stream.window.startTime.getTime(); + } + return parseTs(lastRow); +} + +/** + * The merge frontier: the timestamp every active stream has covered. + * DESC — rows with ts >= frontier are safe to show; ASC — ts <= frontier. + * Null when there are no active streams (nothing bounds the merge). + */ +export function computeFrontier( + streams: StreamSnapshot[], + direction: MergeDirection, + parseTs: (row: Record) => number, +): number | null { + let frontier: number | null = null; + for (const stream of streams) { + if (!stream.isActive) continue; + const covered = coveredUntil(stream, direction, parseTs); + if (frontier == null) { + frontier = covered; + } else { + frontier = + direction === 'DESC' + ? Math.max(frontier, covered) + : Math.min(frontier, covered); + } + } + return frontier; +} + +/** + * The active streams holding the frontier back that can be advanced with + * another page fetch. Streams whose initial fetch hasn't completed are not + * included — their in-flight request IS their advancement. + */ +function laggingStreams( + streams: StreamSnapshot[], + direction: MergeDirection, + parseTs: (row: Record) => number, +): StreamSnapshot[] { + const frontier = computeFrontier(streams, direction, parseTs); + if (frontier == null) return []; + return streams.filter( + stream => + stream.isActive && + stream.hasNextPage && + stream.window != null && + coveredUntil(stream, direction, parseTs) === frontier, + ); +} + +export type MergedRow = Record; + +/** + * Merge all fetched rows across streams into one timestamp-ordered list, + * tagged with their origin source, held back at the frontier. + * + * Rows from inactive (errored/excluded) streams are still included — they are + * valid data — but only active streams bound the frontier, so a dead source + * can't freeze the timeline. + */ +function mergeStreamRows( + streams: StreamSnapshot[], + direction: MergeDirection, + timestampKey: string, +): MergedRow[] { + // Timestamps repeat heavily at second precision; cache the Date parse per + // distinct raw value (same trick as ChartUtils' time-chart transform). + const tsCache = new Map(); + const parseTs = (row: Record): number => { + const raw = row[timestampKey]; + let ts = tsCache.get(raw); + if (ts === undefined) { + ts = new Date(raw).getTime(); + tsCache.set(raw, ts); + } + return ts; + }; + + const frontier = computeFrontier(streams, direction, parseTs); + + const tagged: { row: MergedRow; ts: number }[] = []; + for (const stream of streams) { + for (const row of stream.rows) { + const ts = parseTs(row); + if ( + frontier != null && + (direction === 'DESC' ? ts < frontier : ts > frontier) + ) { + continue; + } + tagged.push({ + row: { + ...row, + [MULTI_SOURCE_ROW_FIELDS.SOURCE_ID]: stream.sourceId, + [MULTI_SOURCE_ROW_FIELDS.SOURCE_NAME]: stream.sourceName, + ...(stream.sourceColor != null + ? { [MULTI_SOURCE_ROW_FIELDS.SOURCE_COLOR]: stream.sourceColor } + : {}), + }, + ts, + }); + } + } + + // Array.prototype.sort is stable, so ties keep (stream order, row order). + tagged.sort((a, b) => (direction === 'DESC' ? b.ts - a.ts : a.ts - b.ts)); + + return tagged.map(t => t.row); +} + +/** + * Convenience wrapper used by the table component: one pass producing the + * merged rows, the frontier (for the "loading up to" indicator), and which + * streams to advance on the next fetch. + */ +export function mergeStreams( + streams: StreamSnapshot[], + direction: MergeDirection, + timestampKey: string, +): { + rows: MergedRow[]; + frontier: number | null; + laggingSourceIds: string[]; +} { + const tsCache = new Map(); + const parseTs = (row: Record): number => { + const raw = row[timestampKey]; + let ts = tsCache.get(raw); + if (ts === undefined) { + ts = new Date(raw).getTime(); + tsCache.set(raw, ts); + } + return ts; + }; + + return { + rows: mergeStreamRows(streams, direction, timestampKey), + frontier: computeFrontier(streams, direction, parseTs), + laggingSourceIds: laggingStreams(streams, direction, parseTs).map( + s => s.sourceId, + ), + }; +} diff --git a/packages/app/src/utils/sourceParams.ts b/packages/app/src/utils/sourceParams.ts index 5874111c99..dc759c11b7 100644 --- a/packages/app/src/utils/sourceParams.ts +++ b/packages/app/src/utils/sourceParams.ts @@ -55,6 +55,51 @@ export type SourceParamResolution = * lowest ID, so the same link always resolves to the same source no matter what * order the API returns them in. */ +/** + * Resolve a list of source params (IDs or names) for multi-source search. + * Each element resolves with the same rules as `resolveSourceParam`; results + * are deduped by ID and capped at `max`. Elements that can't be resolved (or + * resolve to a source of the wrong kind) are reported in `unresolved` so the + * caller can warn without failing the rest of the selection. + */ +export function resolveSourcesParam( + paramValues: string[] | null | undefined, + sources: T[] | undefined, + { kinds, max }: { kinds?: SourceKind[]; max?: number } = {}, +): + | { status: 'pending' } + | { status: 'resolved'; sources: T[]; unresolved: string[] } { + if (paramValues == null || paramValues.length === 0) { + return { status: 'resolved', sources: [], unresolved: [] }; + } + if (sources == null) return { status: 'pending' }; + + const resolved: T[] = []; + const seenIds = new Set(); + const unresolved: string[] = []; + + for (const value of paramValues) { + const resolution = resolveSourceParam(value, sources, { kinds }); + if (resolution.status === 'resolved') { + if (!seenIds.has(resolution.source.id)) { + seenIds.add(resolution.source.id); + resolved.push(resolution.source); + } + } else if ( + resolution.status === 'not-found' || + resolution.status === 'wrong-kind' + ) { + unresolved.push(value); + } + } + + return { + status: 'resolved', + sources: max != null ? resolved.slice(0, max) : resolved, + unresolved, + }; +} + export function resolveSourceParam( paramValue: string | null | undefined, sources: T[] | undefined, diff --git a/packages/common-utils/src/__tests__/clickhouse.test.ts b/packages/common-utils/src/__tests__/clickhouse.test.ts index dcec597f09..f8e2314dc4 100644 --- a/packages/common-utils/src/__tests__/clickhouse.test.ts +++ b/packages/common-utils/src/__tests__/clickhouse.test.ts @@ -185,6 +185,22 @@ describe('chSqlToAliasMap - alias unit test', () => { expect(res).toEqual(aliasMap); }); + it('NULL literal alias (multi-source padding column)', () => { + const chSqlInput: ChSql = { + sql: 'SELECT Timestamp as "__hdx_timestamp", NULL as "__hdx_duration_ms" FROM {HYPERDX_PARAM_1544803905:Identifier}.{HYPERDX_PARAM_129845054:Identifier} ORDER BY Timestamp DESC LIMIT {HYPERDX_PARAM_49586:Int32}', + params: { + HYPERDX_PARAM_1544803905: 'default', + HYPERDX_PARAM_129845054: 'otel_logs', + HYPERDX_PARAM_49586: 200, + }, + }; + const res = chSqlToAliasMap(chSqlInput); + expect(res).toEqual({ + __hdx_timestamp: 'Timestamp', + __hdx_duration_ms: 'NULL', + }); + }); + it('Normal alias, with brackets', () => { const chSqlInput: ChSql = { sql: "SELECT Timestamp as ts,ResourceAttributes['service.name'] as serviceTest,Body,TimestampTime,ServiceName,TimestampTime FROM {HYPERDX_PARAM_1544803905:Identifier}.{HYPERDX_PARAM_129845054:Identifier} WHERE (TimestampTime >= fromUnixTimestamp64Milli({HYPERDX_PARAM_1456399765:Int64}) AND TimestampTime <= fromUnixTimestamp64Milli({HYPERDX_PARAM_1719057412:Int64})) ORDER BY TimestampTime DESC LIMIT {HYPERDX_PARAM_49586:Int32} OFFSET {HYPERDX_PARAM_48:Int32}", diff --git a/packages/common-utils/src/clickhouse/index.ts b/packages/common-utils/src/clickhouse/index.ts index 7e24cd5793..b243f3410c 100644 --- a/packages/common-utils/src/clickhouse/index.ts +++ b/packages/common-utils/src/clickhouse/index.ts @@ -994,6 +994,10 @@ function selectColumnsToAliasMap( `${column.expr.column.expr.value}['${column.expr.array_index[0].index.value}']` : // normal alias column.expr.column.expr.value; + } else if (column.expr.type === 'null') { + // NULL literal projection (multi-source search pads columns a source + // lacks with `NULL AS "alias"`); the parser emits it without a loc. + aliasMap[column.as] = 'NULL'; } else if (column.expr.loc != null) { aliasMap[column.as] = parsedSql.slice( column.expr.loc.start.offset, diff --git a/packages/common-utils/src/core/__tests__/searchChartConfig.test.ts b/packages/common-utils/src/core/__tests__/searchChartConfig.test.ts index 99d28a5c53..32068819d2 100644 --- a/packages/common-utils/src/core/__tests__/searchChartConfig.test.ts +++ b/packages/common-utils/src/core/__tests__/searchChartConfig.test.ts @@ -1,5 +1,7 @@ import { ALERT_COUNT_DEFAULT_SELECT, + buildMultiSourceSearchConfig, + buildMultiSourceSelect, buildSearchChartConfig, } from '@/core/searchChartConfig'; import { DisplayType, Filter, SourceKind, TSource } from '@/types'; @@ -477,3 +479,110 @@ describe('buildSearchChartConfig', () => { }); }); }); + +describe('buildMultiSourceSelect', () => { + it('projects the canonical aliases from a Log source semantic expressions', () => { + const source = makeLogSource({ + displayedTimestampValueExpression: 'Timestamp', + serviceNameExpression: 'ServiceName', + severityTextExpression: 'SeverityText', + bodyExpression: 'Body', + }); + + expect(buildMultiSourceSelect(source)).toBe( + 'Timestamp AS "__hdx_timestamp", ' + + 'ServiceName AS "__hdx_service", ' + + 'SeverityText AS "__hdx_severity", ' + + 'Body AS "__hdx_body"', + ); + }); + + it('falls back to the first timestamp expression and NULL for missing semantics', () => { + const source = makeLogSource({ + timestampValueExpression: 'TimestampTime, Timestamp', + implicitColumnExpression: undefined, + }); + + expect(buildMultiSourceSelect(source)).toBe( + 'TimestampTime AS "__hdx_timestamp", ' + + 'NULL AS "__hdx_service", ' + + 'NULL AS "__hdx_severity", ' + + 'NULL AS "__hdx_body"', + ); + }); + + it('maps Trace sources onto status/span-name and a milliseconds duration', () => { + const source = makeTraceSource({ + serviceNameExpression: 'ServiceName', + statusCodeExpression: 'StatusCode', + spanNameExpression: 'SpanName', + durationExpression: 'Duration', + durationPrecision: 9, + }); + + expect(buildMultiSourceSelect(source, { includeDuration: true })).toBe( + 'Timestamp AS "__hdx_timestamp", ' + + 'ServiceName AS "__hdx_service", ' + + 'StatusCode AS "__hdx_severity", ' + + 'SpanName AS "__hdx_body", ' + + '(Duration)/1e6 AS "__hdx_duration_ms"', + ); + }); + + it('projects NULL duration for Log sources when duration is included', () => { + const source = makeLogSource({ bodyExpression: 'Body' }); + + expect(buildMultiSourceSelect(source, { includeDuration: true })).toContain( + 'NULL AS "__hdx_duration_ms"', + ); + }); + + it('appends extra columns, projecting NULL where a source lacks the column', () => { + const source = makeLogSource({ bodyExpression: 'Body' }); + + const select = buildMultiSourceSelect(source, { + extraColumns: [ + { name: 'ServiceName', expression: 'ServiceName' }, + { name: 'StatusCode', expression: null }, + ], + }); + + expect(select).toContain('ServiceName AS "ServiceName"'); + expect(select).toContain('NULL AS "StatusCode"'); + }); +}); + +describe('buildMultiSourceSearchConfig', () => { + it('keeps the standard search config assembly but swaps in the canonical SELECT', () => { + const source = makeLogSource({ + bodyExpression: 'Body', + tableFilterExpression: "ServiceName != 'noisy'", + }); + + const config = buildMultiSourceSearchConfig(source, { + where: 'error', + whereLanguage: 'lucene', + orderBy: 'TimestampTime DESC', + }); + + expect(config.select).toBe(buildMultiSourceSelect(source)); + expect(config.from).toEqual(source.from); + expect(config.connection).toBe('conn-1'); + expect(config.where).toBe('error'); + expect(config.whereLanguage).toBe('lucene'); + expect(config.orderBy).toBe('TimestampTime DESC'); + // Source-level behaviors (e.g. tableFilterExpression) still apply. + expect(config.filters).toEqual([ + { type: 'sql', condition: "ServiceName != 'noisy'" }, + ]); + }); + + it('never resolves to defaultTableSelectExpression', () => { + const config = buildMultiSourceSearchConfig(makeTraceSource(), { + where: '', + }); + + expect(config.select).not.toContain('SpanName,'); + expect(config.select).toContain('AS "__hdx_timestamp"'); + }); +}); diff --git a/packages/common-utils/src/core/searchChartConfig.ts b/packages/common-utils/src/core/searchChartConfig.ts index 37144db8ab..04d7dcfdca 100644 --- a/packages/common-utils/src/core/searchChartConfig.ts +++ b/packages/common-utils/src/core/searchChartConfig.ts @@ -1,3 +1,4 @@ +import { getFirstTimestampValueExpression } from '@/core/utils'; import { BuilderChartConfig, DateRange, @@ -185,3 +186,143 @@ export function buildSearchChartConfig( return config; } + +/** + * Canonical result-column aliases used when searching across multiple sources + * at once. Every selected source's SELECT is rewritten to this shape, so the + * merged results table can map columns by name regardless of how each source's + * underlying schema names them. + * + * The names are quoted aliases (`expr AS "__hdx_timestamp"`), so ClickHouse + * returns them verbatim — unlike raw expressions, which CH may reformat. + */ +export const MULTI_SOURCE_ALIASES = { + timestamp: '__hdx_timestamp', + service: '__hdx_service', + severity: '__hdx_severity', + body: '__hdx_body', + /** Milliseconds; only projected when a Trace source is in the selection. */ + durationMs: '__hdx_duration_ms', +} as const; + +/** + * An extra user-picked column to project alongside the canonical aliases. + * `expression` is the per-source SQL expression for the column, or null when + * the source has no such column (projected as NULL so every source returns + * the same column set). + */ +export type MultiSourceExtraColumn = { + /** Result column name (used verbatim as the quoted alias). */ + name: string; + expression: string | null; +}; + +const quoteAlias = (name: string) => `"${name.replace(/"/g, '\\"')}"`; + +/** + * Per-source semantic expression for each canonical alias. Mirrors the app's + * display helpers (`getDisplayedTimestampValueExpression`, `getEventBody`, + * `getDurationMsExpression` in packages/app/src/source.ts) — keep in sync. + */ +function multiSourceSemanticExpressions(source: TSource): { + timestamp: string; + service: string; + severity: string; + body: string; + durationMs: string; +} { + const firstTimestamp = getFirstTimestampValueExpression( + source.timestampValueExpression, + ); + + if (isLogSource(source)) { + return { + timestamp: source.displayedTimestampValueExpression || firstTimestamp, + service: source.serviceNameExpression || 'NULL', + severity: source.severityTextExpression || 'NULL', + body: source.bodyExpression || source.implicitColumnExpression || 'NULL', + durationMs: 'NULL', + }; + } + + if (isTraceSource(source)) { + return { + timestamp: source.displayedTimestampValueExpression || firstTimestamp, + service: source.serviceNameExpression || 'NULL', + severity: source.statusCodeExpression || 'NULL', + body: source.spanNameExpression || 'NULL', + // Match getDurationMsExpression: durationPrecision is the sub-second + // digit count (9 = nanoseconds), so /1e(precision-3) yields milliseconds. + durationMs: `(${source.durationExpression})/1e${(source.durationPrecision ?? 9) - 3}`, + }; + } + + // Multi-source search only supports Log and Trace sources today; other kinds + // still get a valid (if minimal) shape so a stray source can't render SQL + // that errors the whole selection. + return { + timestamp: firstTimestamp, + service: 'NULL', + severity: 'NULL', + body: 'NULL', + durationMs: 'NULL', + }; +} + +/** + * Build the canonical aliased SELECT string for one source in a multi-source + * search. Exported for tests. + */ +export function buildMultiSourceSelect( + source: TSource, + { + includeDuration = false, + extraColumns = [], + }: { + /** Project `__hdx_duration_ms` (set when any selected source is a Trace). */ + includeDuration?: boolean; + extraColumns?: MultiSourceExtraColumn[]; + } = {}, +): string { + const exprs = multiSourceSemanticExpressions(source); + + const parts = [ + `${exprs.timestamp} AS ${quoteAlias(MULTI_SOURCE_ALIASES.timestamp)}`, + `${exprs.service} AS ${quoteAlias(MULTI_SOURCE_ALIASES.service)}`, + `${exprs.severity} AS ${quoteAlias(MULTI_SOURCE_ALIASES.severity)}`, + `${exprs.body} AS ${quoteAlias(MULTI_SOURCE_ALIASES.body)}`, + ]; + if (includeDuration) { + parts.push( + `${exprs.durationMs} AS ${quoteAlias(MULTI_SOURCE_ALIASES.durationMs)}`, + ); + } + for (const col of extraColumns) { + parts.push(`${col.expression ?? 'NULL'} AS ${quoteAlias(col.name)}`); + } + + return parts.join(', '); +} + +/** + * Build the chart config for one source of a multi-source search: the standard + * `buildSearchChartConfig` assembly with the SELECT replaced by the canonical + * aliased column set, so every selected source returns the same result shape. + * + * The caller supplies `orderBy` per source (each source's own timestamp-based + * default) — a shared orderBy is meaningless across schemas, and time-window + * pagination requires the first orderBy term to be the source's timestamp. + */ +export function buildMultiSourceSearchConfig( + source: TSource, + input: Omit, + opts: { + includeDuration?: boolean; + extraColumns?: MultiSourceExtraColumn[]; + } = {}, +): SearchChartConfig { + return buildSearchChartConfig(source, { + ...input, + select: buildMultiSourceSelect(source, opts), + }); +} From be07d8f30660b01212ae7474604edc28988dd9a3 Mon Sep 17 00:00:00 2001 From: Tom Alexander Date: Tue, 11 Aug 2026 11:05:37 -0400 Subject: [PATCH 2/3] refactor(app): centralize multi-source hook slots in useMultiSourceSlots Three components were each hand-unrolling slot0..slot4 hook calls to keep a constant hook count across a variable source selection. Rules of hooks allow calling a use*-named function parameter from a custom hook, so one generic useMultiSourceSlots(items, useSlot, opts) now owns the unrolling and the MAX_SEARCH_SOURCES pin; consumers are a single call. Slot hooks return memoized values so the returned array is dependency-list safe. --- .../src/components/MultiSourceRowTable.tsx | 24 ++--- .../src/components/MultiSourceTimeChart.tsx | 87 ++++++++++--------- packages/app/src/defaults.ts | 5 +- .../app/src/hooks/useMultiSourceSearch.ts | 57 ++++++++++-- 4 files changed, 103 insertions(+), 70 deletions(-) diff --git a/packages/app/src/components/MultiSourceRowTable.tsx b/packages/app/src/components/MultiSourceRowTable.tsx index 20c35784b5..df3b5e8f77 100644 --- a/packages/app/src/components/MultiSourceRowTable.tsx +++ b/packages/app/src/components/MultiSourceRowTable.tsx @@ -18,6 +18,7 @@ import { IconAlertTriangle } from '@tabler/icons-react'; import api from '@/api'; import { searchChartConfigDefaults } from '@/defaults'; +import { useMultiSourceSlots } from '@/hooks/useMultiSourceSearch'; import useOffsetPaginatedQuery from '@/hooks/useOffsetPaginatedQuery'; import useRowWhere, { RowWhereResult, WithClause } from '@/hooks/useRowWhere'; import { @@ -73,10 +74,9 @@ type SourceStream = { /** * One source's independent query pipeline: the same * defaults → additional-key SELECT merge → windowed offset pagination → - * row-WHERE machinery as the single-source DBSqlRowTable, packaged per slot. - * - * Always called (fixed hook count — see MAX_SEARCH_SOURCES); unused slots get - * a stub config and stay disabled. + * row-WHERE machinery as the single-source DBSqlRowTable, packaged as a + * useMultiSourceSlots slot hook. Unused slots get a stub config and stay + * disabled. */ function useSourceStream( spec: MultiSourceStreamSpec | undefined, @@ -232,28 +232,20 @@ export default function MultiSourceRowTableWithSidebar({ keepOpenSelector?: string; queryKeyPrefix?: string; }) { - const streamOpts = { + const slots = useMultiSourceSlots(specs, useSourceStream, { enabled, isLive, enableSmallFirstWindow, queryKeyPrefix, - }; - - // Fixed hook slots (MAX_SEARCH_SOURCES = 5): hook count stays constant no - // matter how many sources are selected, so no rules-of-hooks gymnastics. - const slot0 = useSourceStream(specs[0], streamOpts); - const slot1 = useSourceStream(specs[1], streamOpts); - const slot2 = useSourceStream(specs[2], streamOpts); - const slot3 = useSourceStream(specs[3], streamOpts); - const slot4 = useSourceStream(specs[4], streamOpts); + }); const streams = useMemo( () => - [slot0, slot1, slot2, slot3, slot4].filter( + slots.filter( (s): s is SourceStream & { spec: MultiSourceStreamSpec } => s.spec != null, ), - [slot0, slot1, slot2, slot3, slot4], + [slots], ); const snapshots: StreamSnapshot[] = useMemo( diff --git a/packages/app/src/components/MultiSourceTimeChart.tsx b/packages/app/src/components/MultiSourceTimeChart.tsx index afb8dbdfba..d5cc44dc07 100644 --- a/packages/app/src/components/MultiSourceTimeChart.tsx +++ b/packages/app/src/components/MultiSourceTimeChart.tsx @@ -23,6 +23,7 @@ import ChartContainer from '@/components/charts/ChartContainer'; import ChartErrorState from '@/components/charts/ChartErrorState'; import { type ActiveClickPayload, MemoChart } from '@/HDXMultiSeriesTimeChart'; import { useQueriedChartConfig } from '@/hooks/useChartConfig'; +import { useMultiSourceSlots } from '@/hooks/useMultiSourceSearch'; import type { NumberFormat } from '@/types'; import { getMultiSourceColor } from './MultiSourceBadge'; @@ -47,6 +48,13 @@ const STUB_CONFIG: BuilderChartConfigWithDateRange = { dateRange: [new Date(0), new Date(0)], }; +type HistogramSlotState = { + data: ReturnType['data']; + isLoading: boolean; + isError: boolean; + error: Error | null; +}; + function useHistogramSlot( spec: MultiSourceChartSpec | undefined, { @@ -60,37 +68,47 @@ function useHistogramSlot( enableParallelQueries?: boolean; parallelizeWhenPossible?: boolean; }, -) { +): HistogramSlotState { const queriedConfig = useMemo( () => convertToTimeChartConfig(spec?.config ?? STUB_CONFIG), [spec?.config], ); - return useQueriedChartConfig(queriedConfig, { - // Key shape mirrors DBTimeChart/SearchTotalCountChart so TanStack can - // de-dupe the histogram and total-count consumers of the same source. - queryKey: [ - queryKeyPrefix, - queriedConfig, - 'chunked', - { - disableQueryChunking: false, - enableParallelQueries, - parallelizeWhenPossible, - }, - ], - placeholderData: keepPreviousData, - enableQueryChunking: true, - enableParallelQueries: enableParallelQueries && parallelizeWhenPossible, - enabled: enabled && spec != null, - }); + const { data, isLoading, isError, error } = useQueriedChartConfig( + queriedConfig, + { + // Key shape mirrors DBTimeChart/SearchTotalCountChart so TanStack can + // de-dupe the histogram and total-count consumers of the same source. + queryKey: [ + queryKeyPrefix, + queriedConfig, + 'chunked', + { + disableQueryChunking: false, + enableParallelQueries, + parallelizeWhenPossible, + }, + ], + placeholderData: keepPreviousData, + enableQueryChunking: true, + enableParallelQueries: enableParallelQueries && parallelizeWhenPossible, + enabled: enabled && spec != null, + }, + ); + + // Stable identity per content change so the slots array (and everything + // memoized on it) doesn't churn on unrelated renders. + return useMemo( + () => ({ data, isLoading, isError, error: error ?? null }), + [data, isLoading, isError, error], + ); } /** - * Runs one count() histogram query per selected source (fixed hook slots, see - * MAX_SEARCH_SOURCES) and merges the responses into a single response shape - * with a synthetic source-name group column — so the standard time-chart - * transform naturally yields one series per source. + * Runs one count() histogram query per selected source (one hook slot per + * source, see useMultiSourceSlots) and merges the responses into a single + * response shape with a synthetic source-name group column — so the standard + * time-chart transform naturally yields one series per source. */ function useMultiSourceHistogram( specs: MultiSourceChartSpec[], @@ -105,20 +123,12 @@ function useMultiSourceHistogram( }, ) { const { data: me, isLoading: isLoadingMe } = api.useMe(); - const slotOpts = { + const slots = useMultiSourceSlots(specs, useHistogramSlot, { enabled: enabled && !isLoadingMe, queryKeyPrefix, enableParallelQueries, parallelizeWhenPossible: me?.team?.parallelizeWhenPossible, - }; - - const slot0 = useHistogramSlot(specs[0], slotOpts); - const slot1 = useHistogramSlot(specs[1], slotOpts); - const slot2 = useHistogramSlot(specs[2], slotOpts); - const slot3 = useHistogramSlot(specs[3], slotOpts); - const slot4 = useHistogramSlot(specs[4], slotOpts); - - const slots = [slot0, slot1, slot2, slot3, slot4].slice(0, specs.length); + }); const isLoading = slots.some(s => s.isLoading); const allFailed = slots.length > 0 && slots.every(s => s.isError); @@ -129,17 +139,10 @@ function useMultiSourceHistogram( const mergedResponse: ResponseJSON> | undefined = useMemo(() => { - const slotData = [ - slot0.data, - slot1.data, - slot2.data, - slot3.data, - slot4.data, - ]; let meta: ColumnMetaType[] | undefined; const data: Record[] = []; for (let i = 0; i < specs.length; i++) { - const response = slotData[i]; + const response = slots[i]?.data; if (response?.meta == null || response.meta.length === 0) continue; if (meta == null) { meta = [ @@ -153,7 +156,7 @@ function useMultiSourceHistogram( } } return meta ? { data, meta, rows: data.length } : undefined; - }, [slot0.data, slot1.data, slot2.data, slot3.data, slot4.data, specs]); + }, [slots, specs]); return { mergedResponse, isLoading, allFailed, anyError, error, isComplete }; } diff --git a/packages/app/src/defaults.ts b/packages/app/src/defaults.ts index cf850a2fd2..efaeb1b269 100644 --- a/packages/app/src/defaults.ts +++ b/packages/app/src/defaults.ts @@ -6,9 +6,8 @@ export const DEFAULT_SEARCH_ROW_LIMIT = 200; // Ceiling on how many sources one search can span. Each selected source runs // its own result stream plus histogram/count aggregates, so the fan-out cost // is N× a single search; 5 keeps the worst case bounded while covering the -// common "a few log sources + traces" setups. Also the size of the fixed -// hook-slot arrays in the multi-source components — raising it means adding -// slots there too. +// common "a few log sources + traces" setups. Also the hook-slot count in +// useMultiSourceSlots — raising it means adding a slot there too. export const MAX_SEARCH_SOURCES = 5; export const DEFAULT_QUERY_TIMEOUT = 60; // max_execution_time, seconds export const DEFAULT_FILTER_KEYS_FETCH_LIMIT = 100; diff --git a/packages/app/src/hooks/useMultiSourceSearch.ts b/packages/app/src/hooks/useMultiSourceSearch.ts index 9493bbd848..18272b4414 100644 --- a/packages/app/src/hooks/useMultiSourceSearch.ts +++ b/packages/app/src/hooks/useMultiSourceSearch.ts @@ -1,9 +1,43 @@ import { useMemo } from 'react'; +import { ColumnMeta } from '@hyperdx/common-utils/dist/clickhouse'; import { MultiSourceExtraColumn } from '@hyperdx/common-utils/dist/core/searchChartConfig'; import { TSource } from '@hyperdx/common-utils/dist/types'; +import { MAX_SEARCH_SOURCES } from '@/defaults'; import { useColumns } from '@/hooks/useMetadata'; +/** + * Run one instance of a hook per selected source of a multi-source search. + * + * The rules of hooks require a constant hook count per component, but multi + * mode needs one query pipeline per selected source — and `useQueries` can't + * cover these pipelines (the row streams are `useInfiniteQuery`-based, which + * has no plural form, and the chart pipeline composes other hooks). So the + * hook count is pinned at MAX_SEARCH_SOURCES here, in one place: unused + * slots receive `undefined` and every slot hook is expected to self-disable + * for it. + * + * `useSlot` must be a stable, named hook (the rules-of-hooks lint understands + * `use*`-named parameters) and should return a memoized value, so the array + * this returns is referentially stable and safe to use in dependency lists. + */ +export function useMultiSourceSlots( + items: readonly Item[], + useSlot: (item: Item | undefined, opts: Opts) => Result, + opts: Opts, +): Result[] { + const s0 = useSlot(items[0], opts); + const s1 = useSlot(items[1], opts); + const s2 = useSlot(items[2], opts); + const s3 = useSlot(items[3], opts); + const s4 = useSlot(items[4], opts); + const count = Math.min(items.length, MAX_SEARCH_SOURCES); + return useMemo( + () => [s0, s1, s2, s3, s4].slice(0, count), + [s0, s1, s2, s3, s4, count], + ); +} + const EMPTY_SOURCE_PARAMS = { databaseName: '', tableName: '', @@ -25,24 +59,29 @@ export type MultiSourceColumnOption = { availableCount: number; }; +/** Slot hook: DESCRIBE columns for one source. Stable — `.data` is cached. */ +function useSourceColumnsSlot( + source: TSource | undefined, +): ColumnMeta[] | undefined { + return useColumns(columnsParamsFor(source)).data; +} + /** * Top-level columns (DESCRIBE) for each selected source of a multi-source * search, plus the deduped union with per-column availability counts for the - * add-column picker. Fixed hook slots (MAX_SEARCH_SOURCES = 5); useColumns - * self-disables for empty slots. + * add-column picker. useColumns self-disables for unused slots. */ export function useMultiSourceColumns(sources: TSource[]): { columnsBySourceId: Map>; unionColumns: MultiSourceColumnOption[]; } { - const q0 = useColumns(columnsParamsFor(sources[0])); - const q1 = useColumns(columnsParamsFor(sources[1])); - const q2 = useColumns(columnsParamsFor(sources[2])); - const q3 = useColumns(columnsParamsFor(sources[3])); - const q4 = useColumns(columnsParamsFor(sources[4])); + const slotData = useMultiSourceSlots( + sources, + useSourceColumnsSlot, + undefined, + ); return useMemo(() => { - const slotData = [q0.data, q1.data, q2.data, q3.data, q4.data]; const columnsBySourceId = new Map>(); const availability = new Map(); @@ -65,7 +104,7 @@ export function useMultiSourceColumns(sources: TSource[]): { ); return { columnsBySourceId, unionColumns }; - }, [q0.data, q1.data, q2.data, q3.data, q4.data, sources]); + }, [slotData, sources]); } /** Quote a column name as a ClickHouse identifier when it needs it. */ From 0ab39a24de390757d631e71c369d1bf7d9e28d6c Mon Sep 17 00:00:00 2001 From: Tom Alexander Date: Tue, 11 Aug 2026 11:12:58 -0400 Subject: [PATCH 3/3] feat(app): cap multi-source search at 3 sources Query cost scales linearly with the selection (~3 ClickHouse queries per source per refresh, re-fired every live-tail tick, plus one-time metadata), and past 3 sources the interleaved timeline stops being legible. 3 covers the common "app logs + infra logs + traces" case. --- .changeset/multi-source-search.md | 2 +- packages/app/src/defaults.ts | 11 ++++++----- packages/app/src/hooks/useMultiSourceSearch.ts | 7 +------ 3 files changed, 8 insertions(+), 12 deletions(-) diff --git a/.changeset/multi-source-search.md b/.changeset/multi-source-search.md index 9af499a0b7..9d4868f8f4 100644 --- a/.changeset/multi-source-search.md +++ b/.changeset/multi-source-search.md @@ -4,7 +4,7 @@ --- Search across multiple sources at once. The search page's source selector can -now expand into a multi-select (up to 5 log/trace sources): results interleave +now expand into a multi-select (up to 3 log/trace sources): results interleave into one timestamp-ordered timeline with a per-row source badge, normalized columns (Timestamp, Source, Service, Level, Message, and Duration when traces are included), a histogram stacked by source, and an add-column picker over the diff --git a/packages/app/src/defaults.ts b/packages/app/src/defaults.ts index efaeb1b269..5b67f68216 100644 --- a/packages/app/src/defaults.ts +++ b/packages/app/src/defaults.ts @@ -3,12 +3,13 @@ import type { BuilderChartConfigWithDateRange } from '@hyperdx/common-utils/dist // Limit defaults export const DEFAULT_SEARCH_ROW_LIMIT = 200; -// Ceiling on how many sources one search can span. Each selected source runs -// its own result stream plus histogram/count aggregates, so the fan-out cost -// is N× a single search; 5 keeps the worst case bounded while covering the -// common "a few log sources + traces" setups. Also the hook-slot count in +// Ceiling on how many sources one search can span. Cost scales linearly with +// the selection: each source runs its own result stream plus histogram/count +// aggregates (~3 ClickHouse queries per source per refresh, re-fired every +// live-tail tick), so 3 keeps the worst case bounded while covering the +// common "app logs + infra logs + traces" setups. Also the hook-slot count in // useMultiSourceSlots — raising it means adding a slot there too. -export const MAX_SEARCH_SOURCES = 5; +export const MAX_SEARCH_SOURCES = 3; export const DEFAULT_QUERY_TIMEOUT = 60; // max_execution_time, seconds export const DEFAULT_FILTER_KEYS_FETCH_LIMIT = 100; export const DEFAULT_SERIES_LIMIT = 100; diff --git a/packages/app/src/hooks/useMultiSourceSearch.ts b/packages/app/src/hooks/useMultiSourceSearch.ts index 18272b4414..052a0c70ee 100644 --- a/packages/app/src/hooks/useMultiSourceSearch.ts +++ b/packages/app/src/hooks/useMultiSourceSearch.ts @@ -29,13 +29,8 @@ export function useMultiSourceSlots( const s0 = useSlot(items[0], opts); const s1 = useSlot(items[1], opts); const s2 = useSlot(items[2], opts); - const s3 = useSlot(items[3], opts); - const s4 = useSlot(items[4], opts); const count = Math.min(items.length, MAX_SEARCH_SOURCES); - return useMemo( - () => [s0, s1, s2, s3, s4].slice(0, count), - [s0, s1, s2, s3, s4, count], - ); + return useMemo(() => [s0, s1, s2].slice(0, count), [s0, s1, s2, count]); } const EMPTY_SOURCE_PARAMS = {