From 667dfb11d08d19b900234992ec384257043dce13 Mon Sep 17 00:00:00 2001 From: Tom Alexander Date: Tue, 11 Aug 2026 12:06:57 -0400 Subject: [PATCH] feat(app): filters sidebar for multi-source search The filters sidebar now works across a multi-source selection. Each selected source runs its own facet pipeline (fields + values); the sidebar merges them by field path with values unioned, reusing the single-source FilterGroup rendering. Checking a value filters every source that has the field. A source whose table lacks a filtered column is excluded from the search with a visible reason on its status chip, rather than silently returning rows that ignore the filter. Filter pills and add-to-filter from the row side panel are re-enabled in multi mode, with filter-key escaping and date-column detection driven by the union of the selected sources' schemas. --- .changeset/multi-source-filters.md | 10 + packages/app/src/DBSearchPage.tsx | 393 +++++++++++------- .../components/DBSearchPageFilters/hooks.ts | 5 +- .../src/components/MultiSourceRowTable.tsx | 30 +- .../components/MultiSourceSearchFilters.tsx | 315 ++++++++++++++ .../src/components/MultiSourceTimeChart.tsx | 4 +- .../__tests__/useMultiSourceSearch.test.ts | 87 ++++ .../app/src/hooks/useMultiSourceSearch.ts | 56 ++- 8 files changed, 743 insertions(+), 157 deletions(-) create mode 100644 .changeset/multi-source-filters.md create mode 100644 packages/app/src/components/MultiSourceSearchFilters.tsx create mode 100644 packages/app/src/hooks/__tests__/useMultiSourceSearch.test.ts diff --git a/.changeset/multi-source-filters.md b/.changeset/multi-source-filters.md new file mode 100644 index 0000000000..7bb2603e91 --- /dev/null +++ b/.changeset/multi-source-filters.md @@ -0,0 +1,10 @@ +--- +'@hyperdx/app': minor +--- + +The filters sidebar now works when searching multiple sources. Facet fields +and values merge across the selected sources, and checking a value filters +every source that has the field. A source whose table lacks a filtered column +is excluded from the results with a visible reason on its status chip instead +of silently returning unfiltered rows. Filter pills and add-to-filter from the +row side panel work in multi-source mode too. diff --git a/packages/app/src/DBSearchPage.tsx b/packages/app/src/DBSearchPage.tsx index acd284bbed..353137b0f6 100644 --- a/packages/app/src/DBSearchPage.tsx +++ b/packages/app/src/DBSearchPage.tsx @@ -101,6 +101,7 @@ import ResourceTerraformPopover from '@/components/Iac/ResourceTerraformPopover' import { InputControlled } from '@/components/InputControlled'; import MultiSourceColumnPicker from '@/components/MultiSourceColumnPicker'; import MultiSourceRowTableWithSidebar from '@/components/MultiSourceRowTable'; +import MultiSourceSearchFilters from '@/components/MultiSourceSearchFilters'; import { MultiSourceTimeChart, MultiSourceTotalCountChart, @@ -123,6 +124,7 @@ import { useAliasMapFromChartConfig } from '@/hooks/useChartConfig'; import { useExplainQuery } from '@/hooks/useExplainQuery'; import { resolveExtraColumnsForSource, + unresolvedFilterColumns, useMultiSourceColumns, } from '@/hooks/useMultiSourceSearch'; import { useResolvedSourceParam } from '@/hooks/useResolvedSourceParam'; @@ -1351,8 +1353,123 @@ export function DBSearchPage() { [debouncedSubmit, setValue], ); + const watchedSource = useWatch({ + control, + name: 'source', + // Watch will reset when changing saved search, so we need to default to the URL + defaultValue: searchedConfig.source ?? undefined, + }); + + // --- Multi-source search: selection & schema state ------------------------ + // 2+ resolved sources in the ?sources= param engage multi mode: one + // independent query pipeline per source, merged client-side. The single + // `source` (primary) keeps every existing code path working; multi mode + // only swaps what gets rendered below. Declared before the filter-state + // hooks so they can work against the union of the selected schemas. + const { sources: searchedMultiSources } = useResolvedSourcesParam( + rawSearchedConfig.sources, + { kinds: ALLOWED_SOURCE_KINDS }, + ); + const isMultiSource = searchedMultiSources.length > 1; + // Delta/pattern analyses are per-source; multi mode pins the results view. + const effectiveAnalysisMode = isMultiSource ? 'results' : analysisMode; + // Raw SQL WHERE names concrete columns of a concrete table — reinterpreting + // it per source risks silently-wrong results, so multi mode requires Lucene. + const isMultiSourceSqlBlocked = + isMultiSource && + (searchedConfig.whereLanguage ?? getStoredLanguage() ?? 'lucene') === + 'sql' && + !!searchedConfig.where; + + // A hand-authored URL may carry only ?sources=; backfill the primary so the + // single-source machinery (form, chart config) has one. + useEffect(() => { + if (!rawSearchedConfig.source && searchedMultiSources.length > 0) { + setSearchedConfig({ source: searchedMultiSources[0].id }); + } + }, [rawSearchedConfig.source, searchedMultiSources, setSearchedConfig]); + + const watchedSources = useWatch({ control, name: 'sources' }); + const formSourceCount = watchedSources?.length ?? 0; + // The multi-select UI stays visible while the user is composing a selection + // (even before a second source is added). + const [multiPickerOpen, setMultiPickerOpen] = useState(false); + const isMultiSelectUI = multiPickerOpen || formSourceCount > 1; + const formIsMulti = formSourceCount > 1; + + const enterMultiSourceSelect = useCallback(() => { + setValue('sources', watchedSource ? [watchedSource] : []); + setMultiPickerOpen(true); + }, [setValue, watchedSource]); + + // Keep the primary `source` field in sync with the selection and re-run the + // search when the selection changes. + const prevWatchedSourcesRef = useRef(null); + useEffect(() => { + const current = watchedSources ?? []; + const prev = prevWatchedSourcesRef.current; + if (prev != null && JSON.stringify(prev) === JSON.stringify(current)) { + return; + } + prevWatchedSourcesRef.current = current; + if (prev == null) { + // Initial hydration from the URL — nothing changed. + return; + } + if (current.length > 0 && current[0] !== watchedSource) { + setValue('source', current[0]); + } + if ((prev?.length ?? 0) <= 1 && current.length > 1) { + // Entering multi mode: the single-source SELECT/ORDER BY strings don't + // translate to the canonical multi-source shape. + setValue('select', ''); + setValue('orderBy', ''); + } + debouncedSubmit(); + }, [watchedSources, watchedSource, setValue, debouncedSubmit]); + + // Collapse the picker back to the single-source select when the user + // reduces a real multi selection to one source. Keyed on the >1 → ≤1 + // transition so it can't fire in the just-opened composing state (picker + // open, one source selected, second not yet picked). + const prevFormSourceCountRef = useRef(formSourceCount); + useEffect(() => { + const prev = prevFormSourceCountRef.current; + prevFormSourceCountRef.current = formSourceCount; + if (prev > 1 && formSourceCount <= 1) { + setMultiPickerOpen(false); + } + }, [formSourceCount]); + + // Per-source top-level columns: powers the add-column picker, the + // per-source `column vs NULL` projection for user-picked extras, and + // per-source filter resolvability. Driven by the form's draft selection + // while composing (so the picker has options before the search is + // submitted), falling back to the searched selection. + const formMultiSources = useMemo( + () => + (watchedSources ?? []) + .map(id => inputSourceObjs?.find(s => s.id === id)) + .filter((s): s is TSource => s != null), + [watchedSources, inputSourceObjs], + ); + const { + columnsBySourceId, + unionColumns, + dateTimeColumns: multiDateTimeColumns, + } = useMultiSourceColumns( + formMultiSources.length > 1 + ? formMultiSources + : isMultiSource + ? searchedMultiSources + : [], + ); + // --- End multi-source selection & schema state ----------------------------- + // Top-level column names for the active source, used to quote - // filter keys that contain special characters. + // filter keys that contain special characters. In multi mode this is the + // union across the selected sources, so filter keys from any of them + // escape correctly. const { data: inputSourceColumns } = useColumns( { databaseName: inputSourceObj?.from?.databaseName ?? '', @@ -1361,20 +1478,19 @@ export function DBSearchPage() { }, { enabled: !!inputSourceObj }, ); - const knownColumns = useMemo( - () => - inputSourceColumns - ? new Set(inputSourceColumns.map(c => c.name)) - : new Set(), - [inputSourceColumns], - ); + const knownColumns = useMemo(() => { + if (isMultiSource) { + const union = new Set(); + for (const names of columnsBySourceId.values()) { + for (const name of names) union.add(name); + } + return union; + } + return inputSourceColumns + ? new Set(inputSourceColumns.map(c => c.name)) + : new Set(); + }, [inputSourceColumns, isMultiSource, columnsBySourceId]); - const watchedSource = useWatch({ - control, - name: 'source', - // Watch will reset when changing saved search, so we need to default to the URL - defaultValue: searchedConfig.source ?? undefined, - }); const prevSourceRef = useRef(watchedSource); // Set when the user switches sources via the dropdown. The follow-up // effect waits for the new source's columns to load and then drops any @@ -1397,11 +1513,21 @@ export function DBSearchPage() { const { dateTimeColumns, onResolvedColumnsChange } = useResolvedDateTimeColumns(inputSourceColumns); + // In multi mode, date/time-typed filter keys may come from any selected + // source's schema. + const effectiveDateTimeColumns = useMemo( + () => + isMultiSource && multiDateTimeColumns.size > 0 + ? new Map([...dateTimeColumns, ...multiDateTimeColumns]) + : dateTimeColumns, + [isMultiSource, dateTimeColumns, multiDateTimeColumns], + ); + const filters = useWatch({ name: 'filters', control }); const searchFilters = useSearchPageFilterState({ searchQuery: filters ?? undefined, onFilterChange: handleSetFilters, - dateTimeColumns, + dateTimeColumns: effectiveDateTimeColumns, knownColumns, }); @@ -1539,105 +1665,7 @@ export function DBSearchPage() { const { data: chartConfig, isLoading: isChartConfigLoading } = useSearchedConfigToChartConfig(chartSearchConfig, defaultSearchConfig); - // --- Multi-source search ------------------------------------------------- - // 2+ resolved sources in the ?sources= param engage multi mode: one - // independent query pipeline per source, merged client-side. The single - // `source` (primary) keeps every existing code path working; multi mode - // only swaps what gets rendered below. - const { sources: searchedMultiSources } = useResolvedSourcesParam( - rawSearchedConfig.sources, - { kinds: ALLOWED_SOURCE_KINDS }, - ); - const isMultiSource = searchedMultiSources.length > 1; - // Delta/pattern analyses are per-source; multi mode pins the results view. - const effectiveAnalysisMode = isMultiSource ? 'results' : analysisMode; - // Raw SQL WHERE names concrete columns of a concrete table — reinterpreting - // it per source risks silently-wrong results, so multi mode requires Lucene. - const isMultiSourceSqlBlocked = - isMultiSource && - (searchedConfig.whereLanguage ?? getStoredLanguage() ?? 'lucene') === - 'sql' && - !!searchedConfig.where; - - // A hand-authored URL may carry only ?sources=; backfill the primary so the - // single-source machinery (form, chart config) has one. - useEffect(() => { - if (!rawSearchedConfig.source && searchedMultiSources.length > 0) { - setSearchedConfig({ source: searchedMultiSources[0].id }); - } - }, [rawSearchedConfig.source, searchedMultiSources, setSearchedConfig]); - - const watchedSources = useWatch({ control, name: 'sources' }); - const formSourceCount = watchedSources?.length ?? 0; - // The multi-select UI stays visible while the user is composing a selection - // (even before a second source is added). - const [multiPickerOpen, setMultiPickerOpen] = useState(false); - const isMultiSelectUI = multiPickerOpen || formSourceCount > 1; - const formIsMulti = formSourceCount > 1; - - const enterMultiSourceSelect = useCallback(() => { - setValue('sources', watchedSource ? [watchedSource] : []); - setMultiPickerOpen(true); - }, [setValue, watchedSource]); - - // Keep the primary `source` field in sync with the selection and re-run the - // search when the selection changes. - const prevWatchedSourcesRef = useRef(null); - useEffect(() => { - const current = watchedSources ?? []; - const prev = prevWatchedSourcesRef.current; - if (prev != null && JSON.stringify(prev) === JSON.stringify(current)) { - return; - } - prevWatchedSourcesRef.current = current; - if (prev == null) { - // Initial hydration from the URL — nothing changed. - return; - } - if (current.length > 0 && current[0] !== watchedSource) { - setValue('source', current[0]); - } - if ((prev?.length ?? 0) <= 1 && current.length > 1) { - // Entering multi mode: the single-source SELECT/ORDER BY strings don't - // translate to the canonical multi-source shape. - setValue('select', ''); - setValue('orderBy', ''); - } - debouncedSubmit(); - }, [watchedSources, watchedSource, setValue, debouncedSubmit]); - - // Collapse the picker back to the single-source select when the user - // reduces a real multi selection to one source. Keyed on the >1 → ≤1 - // transition so it can't fire in the just-opened composing state (picker - // open, one source selected, second not yet picked). - const prevFormSourceCountRef = useRef(formSourceCount); - useEffect(() => { - const prev = prevFormSourceCountRef.current; - prevFormSourceCountRef.current = formSourceCount; - if (prev > 1 && formSourceCount <= 1) { - setMultiPickerOpen(false); - } - }, [formSourceCount]); - - // Per-source top-level columns: powers the add-column picker and the - // per-source `column vs NULL` projection for user-picked extras. Driven by - // the form's draft selection while composing (so the picker has options - // before the search is submitted), falling back to the searched selection. - const formMultiSources = useMemo( - () => - (watchedSources ?? []) - .map(id => inputSourceObjs?.find(s => s.id === id)) - .filter((s): s is TSource => s != null), - [watchedSources, inputSourceObjs], - ); - const { columnsBySourceId, unionColumns } = useMultiSourceColumns( - formMultiSources.length > 1 - ? formMultiSources - : isMultiSource - ? searchedMultiSources - : [], - ); - + // --- Multi-source search: query specs ------------------------------------- // In multi mode the `select` param holds the extra column names picked by // the user (the canonical columns are always projected). const multiExtraColumnNames = useMemo( @@ -1661,13 +1689,46 @@ export function DBSearchPage() { [setValue, debouncedSubmit], ); + // Sidebar filters apply per source. A source whose table lacks a filtered + // column can't answer the filtered search — it's excluded entirely (with a + // visible reason on its status chip) rather than silently returning rows + // that ignore the filter. + const multiSourceFilters = useMemo( + () => (isMultiSource ? (searchedConfig.filters ?? []) : []), + [isMultiSource, searchedConfig.filters], + ); + const multiDisabledReasons = useMemo(() => { + const reasons = new Map(); + if (!isMultiSource || multiSourceFilters.length === 0) return reasons; + for (const source of searchedMultiSources) { + const missing = unresolvedFilterColumns( + multiSourceFilters, + columnsBySourceId.get(source.id), + ); + if (missing.length > 0) { + reasons.set( + source.id, + `${source.name} is excluded: the active filter uses ${missing.join( + ', ', + )}, which it doesn't have`, + ); + } + } + return reasons; + }, [ + isMultiSource, + multiSourceFilters, + searchedMultiSources, + columnsBySourceId, + ]); + const multiSourceStreamSpecs = useMemo(() => { if (!isMultiSource || isMultiSourceSqlBlocked) return []; - // Extra columns need each source's DESCRIBE to resolve column-vs-NULL; - // hold the row queries until they've loaded so we don't fire a throwaway - // query per source with every extra projected as NULL. + // Extra columns and filters both need each source's DESCRIBE (to resolve + // column-vs-NULL and filter resolvability); hold the row queries until + // they've loaded so we don't fire throwaway or erroring queries. if ( - multiExtraColumnNames.length > 0 && + (multiExtraColumnNames.length > 0 || multiSourceFilters.length > 0) && columnsBySourceId.size < searchedMultiSources.length ) { return []; @@ -1676,12 +1737,14 @@ export function DBSearchPage() { const where = searchedConfig.where ?? ''; return searchedMultiSources.map(source => ({ source, + disabledReason: multiDisabledReasons.get(source.id), config: { ...buildMultiSourceSearchConfig( source, { where, whereLanguage: 'lucene', + filters: multiSourceFilters, orderBy: multiSourceDefaultOrderBy(source), }, { @@ -1701,19 +1764,29 @@ export function DBSearchPage() { searchedMultiSources, searchedConfig.where, multiExtraColumnNames, + multiSourceFilters, + multiDisabledReasons, columnsBySourceId, searchedTimeRange, ]); const multiHistogramSpecs = useMemo(() => { if (!isMultiSource || isMultiSourceSqlBlocked) return []; + if ( + multiSourceFilters.length > 0 && + columnsBySourceId.size < searchedMultiSources.length + ) { + return []; + } const where = searchedConfig.where ?? ''; return searchedMultiSources.map(source => ({ source, + disabledReason: multiDisabledReasons.get(source.id), config: { ...buildMultiSourceSearchConfig(source, { where, whereLanguage: 'lucene', + filters: multiSourceFilters, }), select: ALERT_COUNT_DEFAULT_SELECT, orderBy: undefined, @@ -1731,6 +1804,9 @@ export function DBSearchPage() { isMultiSourceSqlBlocked, searchedMultiSources, searchedConfig.where, + multiSourceFilters, + multiDisabledReasons, + columnsBySourceId, searchedTimeRange, ]); // --- End multi-source search --------------------------------------------- @@ -2218,18 +2294,26 @@ export function DBSearchPage() { ], ); - // Multi-source rows span schemas, so single-source affordances - // (add-to-search, column toggles) are omitted — the side panel hides them. - // Passing no `source` with no toggle callbacks is required: with a null - // context source, deriveRowSidePanelContextForSource treats every row as - // same-source and would leave stale single-source callbacks enabled. + // Multi-source rows span schemas, so the single-source column toggles are + // omitted — the side panel hides them. Property-add-to-filter IS wired: + // filters resolve per source, and a source that lacks the column is + // excluded with a visible reason. Passing no `source` is required: with a + // null context source, deriveRowSidePanelContextForSource treats every row + // as same-source, which is exactly the cross-source semantics filters now + // have. const multiRowTableContext = useMemo( () => ({ + onPropertyAddClick: searchFilters.setFilterValue, generateSearchUrl, isChildModalOpen: isDrawerChildModalOpen, setChildModalOpen: setDrawerChildModalOpen, }), - [generateSearchUrl, isDrawerChildModalOpen, setDrawerChildModalOpen], + [ + searchFilters.setFilterValue, + generateSearchUrl, + isDrawerChildModalOpen, + setDrawerChildModalOpen, + ], ); const inputSourceTableConnection = useMemo( @@ -2693,14 +2777,12 @@ export function DBSearchPage() { - {!isMultiSource && ( - - )} + {searchedConfig != null && searchedSource != null && ( + {!isFilterSidebarCollapsed && + isMultiSource && + !isMultiSourceSqlBlocked && ( + + setIsFilterSidebarCollapsed(true)} + /> + + )} {!isFilterSidebarCollapsed && !isMultiSource && ( - - - {(searchedConfig.filters?.length ?? 0) > 0 && ( - - Sidebar filters aren't applied when searching - multiple sources - - )} - {shouldShowLiveModeHint && ( - + {isFilterSidebarCollapsed && ( + + setIsFilterSidebarCollapsed(false) + } /> )} + + {shouldShowLiveModeHint && ( + + )} { if (stream.spec == null) return null; const name = stream.spec.source.name; + const disabledReason = stream.spec.disabledReason; return ( - + {stream.isFetching && } + {disabledReason != null && ( + + + + + + )} {stream.isError && ( ({ facets: data.keyValues, isLoading, isFetching }), + [data.keyValues, isLoading, isFetching], + ); +} + +const NOOP = () => { + /* pins and load-more are single-source affordances; no-op in multi mode */ +}; +const VALUE_PINS = { onPinClick: NOOP, isPinned: () => false }; + +/** + * Multi-source variant of the search filters sidebar: one facet pipeline per + * selected source, merged by field path with values unioned. Filters apply + * per source; a source that lacks a filtered column is excluded from the + * search (surfaced as a chip on the results table). + * + * Single-source-only affordances (pins, shared filters, value counts, + * load-more, analysis-mode tabs, denoising) are intentionally absent. + */ +export default function MultiSourceSearchFilters({ + specs, + dateRange, + isLive, + knownColumns, + searchFilters, + onCollapse, +}: { + specs: MultiSourceFilterSpec[]; + dateRange: [Date, Date]; + isLive: boolean; + /** Union of the selected sources' top-level column names (for escaping). */ + knownColumns: Set; + searchFilters: FilterStateHook; + onCollapse?: () => void; +}) { + const { size, startResize } = useResizable(16, 'left'); + const { + filters: filterState, + setFilterValue, + clearFilter, + clearAllFilters, + setFilterRange, + } = searchFilters; + + const slots = useMultiSourceSlots(specs, useSourceFacetsSlot, { + dateRange, + filterState, + }); + + const isFetching = slots.some(s => s.isFetching); + const isLoading = slots.some(s => s.isLoading); + + // Merge facets across sources: union values per field path, in first-seen + // order (the first selected source's ordering wins). + const mergedFacets = useMemo(() => { + const byKey = new Map }>(); + for (const slot of slots) { + for (const facet of slot.facets ?? []) { + let entry = byKey.get(facet.key); + if (entry == null) { + entry = { values: [], seen: new Set() }; + byKey.set(facet.key, entry); + } + for (const value of facet.value) { + if (!entry.seen.has(value)) { + entry.seen.add(value); + entry.values.push(value); + } + } + } + } + return [...byKey.entries()].map(([key, entry]) => ({ + key, + value: entry.values, + })); + }, [slots]); + + const hasSelections = Object.keys(filterState).length > 0; + const firstConfig = specs[0]?.config ?? STUB_CONFIG; + const { grouped, nonGrouped } = useMemo( + () => groupFacetsByBaseName(mergedFacets), + [mergedFacets], + ); + + return ( + +
+ + + + + Filters {isFetching && '···'} + + + {hasSelections && ( + + + + + + )} + {onCollapse && ( + + + + + + )} + + + + Values across all selected sources. A filter on a field a source + doesn't have excludes that source from the results. + + {grouped.map(group => ( + ({ + ...child, + sqlKey: toQuotedClickHouseKeyExpression( + child.key, + knownColumns, + ), + }))} + selectedValues={group.children.reduce((acc, child) => { + acc[child.key] = getFilterStateEntry( + filterState, + child.key, + ) ?? { + included: new Set(), + excluded: new Set(), + }; + return acc; + }, {} as FilterState)} + onChange={(key, value) => setFilterValue(key, value)} + onClearClick={key => clearFilter(key)} + onOnlyClick={(key, value) => setFilterValue(key, value, 'only')} + onExcludeClick={(key, value) => + setFilterValue(key, value, 'exclude') + } + onPinClick={NOOP} + isPinned={() => false} + showFilterCounts={false} + onLoadMore={NOOP} + loadMoreLoading={{}} + hasLoadedMore={{}} + isDefaultExpanded={group.children.some(child => { + const entry = getFilterStateEntry(filterState, child.key); + return ( + entry != null && + (entry.included.size > 0 || entry.excluded.size > 0) + ); + })} + chartConfig={firstConfig} + isLive={isLive} + /> + ))} + {nonGrouped.map(facet => { + const facetSqlKey = toQuotedClickHouseKeyExpression( + facet.key, + knownColumns, + ); + const entry = getFilterStateEntry(filterState, facet.key); + return ( + ({ + value, + label: value.toString(), + }))} + optionsLoading={isLoading} + selectedValues={ + entry ?? { included: new Set(), excluded: new Set() } + } + onChange={value => setFilterValue(facet.key, value)} + onClearClick={() => clearFilter(facet.key)} + onOnlyClick={value => setFilterValue(facet.key, value, 'only')} + onExcludeClick={value => + setFilterValue(facet.key, value, 'exclude') + } + valuePins={VALUE_PINS} + onLoadMore={NOOP} + loadMoreLoading={false} + hasLoadedMore={false} + isDefaultExpanded={ + entry != null && + (entry.included.size > 0 || + entry.excluded.size > 0 || + entry.range != null) + } + chartConfig={firstConfig} + isLive={isLive} + onRangeChange={range => setFilterRange(facet.key, range)} + /> + ); + })} + {!isLoading && mergedFacets.length === 0 && ( + + No filterable fields found. + + )} + + + + ); +} diff --git a/packages/app/src/components/MultiSourceTimeChart.tsx b/packages/app/src/components/MultiSourceTimeChart.tsx index d5cc44dc07..122c937c49 100644 --- a/packages/app/src/components/MultiSourceTimeChart.tsx +++ b/packages/app/src/components/MultiSourceTimeChart.tsx @@ -35,6 +35,8 @@ export type MultiSourceChartSpec = { source: TSource; /** Per-source count() histogram config (canonical WHERE, no groupBy). */ config: BuilderChartConfigWithDateRange; + /** When set, the source doesn't run (mirrors MultiSourceStreamSpec). */ + disabledReason?: string; }; // Placeholder for unused hook slots; never queried (enabled: false). @@ -92,7 +94,7 @@ function useHistogramSlot( placeholderData: keepPreviousData, enableQueryChunking: true, enableParallelQueries: enableParallelQueries && parallelizeWhenPossible, - enabled: enabled && spec != null, + enabled: enabled && spec != null && spec.disabledReason == null, }, ); diff --git a/packages/app/src/hooks/__tests__/useMultiSourceSearch.test.ts b/packages/app/src/hooks/__tests__/useMultiSourceSearch.test.ts new file mode 100644 index 0000000000..58be8d33d2 --- /dev/null +++ b/packages/app/src/hooks/__tests__/useMultiSourceSearch.test.ts @@ -0,0 +1,87 @@ +import { Filter } from '@hyperdx/common-utils/dist/types'; + +import { + filterRootColumn, + resolveExtraColumnsForSource, + unresolvedFilterColumns, +} from '@/hooks/useMultiSourceSearch'; + +describe('filterRootColumn', () => { + it('extracts a plain column reference', () => { + const filter: Filter = { + type: 'sql_ast', + operator: '=', + left: 'ServiceName', + right: "'cart'", + }; + expect(filterRootColumn(filter)).toBe('ServiceName'); + }); + + it('extracts the root of a map subscript', () => { + const filter: Filter = { + type: 'sql_ast', + operator: '=', + left: "LogAttributes['level']", + right: "'error'", + }; + expect(filterRootColumn(filter)).toBe('LogAttributes'); + }); + + it('extracts a backticked identifier', () => { + const filter: Filter = { + type: 'sql_ast', + operator: '=', + left: '`weird-col`', + right: "'x'", + }; + expect(filterRootColumn(filter)).toBe('weird-col'); + }); + + it('returns null for raw sql and lucene filters', () => { + expect( + filterRootColumn({ type: 'sql', condition: "Foo = 'bar'" }), + ).toBeNull(); + expect( + filterRootColumn({ type: 'lucene', condition: 'foo:bar' }), + ).toBeNull(); + }); +}); + +describe('unresolvedFilterColumns', () => { + const filters: Filter[] = [ + { type: 'sql_ast', operator: '=', left: 'ServiceName', right: "'cart'" }, + { type: 'sql_ast', operator: '=', left: 'StatusCode', right: "'Unset'" }, + { type: 'sql', condition: 'anything' }, + ]; + + it('reports columns the source lacks', () => { + expect( + unresolvedFilterColumns(filters, new Set(['ServiceName', 'Body'])), + ).toEqual(['StatusCode']); + }); + + it('is empty when every attributable column resolves', () => { + expect( + unresolvedFilterColumns(filters, new Set(['ServiceName', 'StatusCode'])), + ).toEqual([]); + }); + + it('is empty (not excluding) while columns are still unknown', () => { + expect(unresolvedFilterColumns(filters, undefined)).toEqual([]); + }); +}); + +describe('resolveExtraColumnsForSource', () => { + it('projects the column where present and NULL where missing', () => { + expect( + resolveExtraColumnsForSource( + ['ServiceName', 'StatusCode', 'weird col'], + new Set(['ServiceName', 'weird col']), + ), + ).toEqual([ + { name: 'ServiceName', expression: 'ServiceName' }, + { name: 'StatusCode', expression: null }, + { name: 'weird col', expression: '`weird col`' }, + ]); + }); +}); diff --git a/packages/app/src/hooks/useMultiSourceSearch.ts b/packages/app/src/hooks/useMultiSourceSearch.ts index 052a0c70ee..691cadc9bf 100644 --- a/packages/app/src/hooks/useMultiSourceSearch.ts +++ b/packages/app/src/hooks/useMultiSourceSearch.ts @@ -1,7 +1,11 @@ import { useMemo } from 'react'; -import { ColumnMeta } from '@hyperdx/common-utils/dist/clickhouse'; +import { + ColumnMeta, + filterColumnMetaByType, + JSDataType, +} from '@hyperdx/common-utils/dist/clickhouse'; import { MultiSourceExtraColumn } from '@hyperdx/common-utils/dist/core/searchChartConfig'; -import { TSource } from '@hyperdx/common-utils/dist/types'; +import { Filter, TSource } from '@hyperdx/common-utils/dist/types'; import { MAX_SEARCH_SOURCES } from '@/defaults'; import { useColumns } from '@/hooks/useMetadata'; @@ -69,6 +73,8 @@ function useSourceColumnsSlot( export function useMultiSourceColumns(sources: TSource[]): { columnsBySourceId: Map>; unionColumns: MultiSourceColumnOption[]; + /** Union of Date/DateTime column name → ClickHouse type across sources. */ + dateTimeColumns: Map; } { const slotData = useMultiSourceSlots( sources, @@ -79,6 +85,7 @@ export function useMultiSourceColumns(sources: TSource[]): { return useMemo(() => { const columnsBySourceId = new Map>(); const availability = new Map(); + const dateTimeColumns = new Map(); for (let i = 0; i < sources.length; i++) { const source = sources[i]; @@ -89,6 +96,12 @@ export function useMultiSourceColumns(sources: TSource[]): { for (const name of names) { availability.set(name, (availability.get(name) ?? 0) + 1); } + for (const col of filterColumnMetaByType(columns, [JSDataType.Date]) ?? + []) { + if (!dateTimeColumns.has(col.name)) { + dateTimeColumns.set(col.name, col.type); + } + } } const unionColumns = [...availability.entries()] @@ -98,10 +111,47 @@ export function useMultiSourceColumns(sources: TSource[]): { b.availableCount - a.availableCount || a.name.localeCompare(b.name), ); - return { columnsBySourceId, unionColumns }; + return { columnsBySourceId, unionColumns, dateTimeColumns }; }, [slotData, sources]); } +/** + * Root column a filter references, for per-source resolvability checks. + * sql_ast filters carry the escaped SQL key in `left` (e.g. `ServiceName`, + * a backticked identifier, or `LogAttributes['level']` whose root is + * `LogAttributes`). Other filter types (raw sql/lucene conditions) can't be + * attributed to a single column and return null — callers should apply them + * to every source and rely on per-source error isolation. + */ +export function filterRootColumn(filter: Filter): string | null { + if (filter.type !== 'sql_ast') return null; + const left = filter.left.trim(); + const backticked = left.match(/^`([^`]+)`/); + if (backticked) return backticked[1]; + const plain = left.match(/^[A-Za-z_][A-Za-z0-9_]*/); + return plain ? plain[0] : null; +} + +/** + * For one source: which of the active filters reference a column its table + * doesn't have. A non-empty result means the source can't answer the + * filtered search and should be excluded (with a visible reason). + */ +export function unresolvedFilterColumns( + filters: Filter[], + sourceColumns: Set | undefined, +): string[] { + if (sourceColumns == null) return []; + const missing = new Set(); + for (const filter of filters) { + const root = filterRootColumn(filter); + if (root != null && !sourceColumns.has(root)) { + missing.add(root); + } + } + return [...missing]; +} + /** Quote a column name as a ClickHouse identifier when it needs it. */ function quoteIdentifier(name: string): string { return /^[A-Za-z_][A-Za-z0-9_]*$/.test(name)