diff --git a/.changeset/multi-source-filters.md b/.changeset/multi-source-filters.md new file mode 100644 index 0000000000..d933bb2629 --- /dev/null +++ b/.changeset/multi-source-filters.md @@ -0,0 +1,12 @@ +--- +'@hyperdx/app': minor +--- + +The filters sidebar works across every selected source. Facet fields and +values merge across sources, value counts are summed, "load more" fans out, +and pins (personal and team-shared) read as a union and apply to the whole +selection. Checking a value filters every source that has the field; a source +whose table lacks a filtered column is excluded from the results with a +visible reason on its status chip instead of silently returning unfiltered +rows. Filter pills and add-to-filter from the row side panel work across +sources too. diff --git a/packages/app/src/DBSearchPage.tsx b/packages/app/src/DBSearchPage.tsx index c11ff3efe1..caa99bf8df 100644 --- a/packages/app/src/DBSearchPage.tsx +++ b/packages/app/src/DBSearchPage.tsx @@ -32,6 +32,7 @@ import { } from '@hyperdx/common-utils/dist/clickhouse'; import { tcFromSource } from '@hyperdx/common-utils/dist/core/metadata'; import { + ALERT_COUNT_DEFAULT_SELECT, buildMultiSourceSearchConfig, buildSearchChartConfig, } from '@hyperdx/common-utils/dist/core/searchChartConfig'; @@ -92,7 +93,7 @@ import { AlertStatusIcon } from '@/components/AlertStatusIcon'; import { ContactSupportText } from '@/components/ContactSupportText'; import { DBSearchPageFilters } from '@/components/DBSearchPageFilters'; import { cleanClickHouseExpression } from '@/components/DBSearchPageFilters/utils'; -import { DBTimeChart, type SeriesGroupFilter } from '@/components/DBTimeChart'; +import { type SeriesGroupFilter } from '@/components/DBTimeChart'; import EmptyState from '@/components/EmptyState'; import { ErrorBoundary } from '@/components/Error/ErrorBoundary'; import { FavoriteButton } from '@/components/FavoriteButton'; @@ -100,12 +101,16 @@ import ResourceTerraformPopover from '@/components/Iac/ResourceTerraformPopover' import { InputControlled } from '@/components/InputControlled'; import MultiSourceColumnPicker from '@/components/MultiSourceColumnPicker'; import OnboardingModal from '@/components/OnboardingModal'; +import { + SearchHistogram, + type SearchHistogramSpec, + SearchTotalCount, +} from '@/components/SearchHistogram'; import SearchWhereInput, { getStoredLanguage, } from '@/components/SearchInput/SearchWhereInput'; import SearchPageActionBar from '@/components/SearchPageActionBar'; import SearchResultsTable from '@/components/SearchResultsTable'; -import SearchTotalCountChart from '@/components/SearchTotalCountChart'; import { SourceMultiSelectControlled } from '@/components/SourceMultiSelect'; import { TableSourceForm } from '@/components/Sources/SourceForm'; import { SourceSelectControlled } from '@/components/SourceSelect'; @@ -343,12 +348,12 @@ function ExpandFiltersButton({ onExpand }: { onExpand: () => void }) { function SearchResultsCountGroup({ isFilterSidebarCollapsed, onExpandFilters, - histogramTimeChartConfig, + histogramSpecs, enableParallelQueries, }: { isFilterSidebarCollapsed: boolean; onExpandFilters: () => void; - histogramTimeChartConfig: BuilderChartConfigWithDateRange; + histogramSpecs: SearchHistogramSpec[]; enableParallelQueries?: boolean; }) { return ( @@ -356,8 +361,8 @@ function SearchResultsCountGroup({ {isFilterSidebarCollapsed && ( )} - @@ -1785,6 +1790,45 @@ export function DBSearchPage() { searchedTimeRange, ]); + const multiHistogramSpecs = useMemo(() => { + if (!isMultiSource || isMultiSourceSqlBlocked) return []; + if ( + multiSourceFilters.length > 0 && + columnsBySourceId.size < searchedMultiSources.length + ) { + return []; + } + const where = searchedConfig.where ?? ''; + return searchedMultiSources.map(source => ({ + source, + disabledReason: multiDisabledReasons.get(source.id), + config: { + ...buildMultiSourceSearchConfig(source, { + where, + whereLanguage: 'lucene', + filters: multiSourceFilters, + }), + select: ALERT_COUNT_DEFAULT_SELECT, + orderBy: undefined, + granularity: 'auto' as const, + dateRange: searchedTimeRange, + displayType: DisplayType.StackedBar, + // Match the single-source histogram: reflect the user's exact range + // so chart and table counts agree (see histogramTimeChartConfig). + alignDateRangeToGranularity: false, + dateRangeEndInclusive: true, + }, + })); + }, [ + isMultiSource, + isMultiSourceSqlBlocked, + searchedMultiSources, + searchedConfig.where, + multiSourceFilters, + multiDisabledReasons, + columnsBySourceId, + searchedTimeRange, + ]); // --- End multi-source search --------------------------------------------- // query error handling @@ -2150,6 +2194,20 @@ export function DBSearchPage() { searchedConfig.select, ]); + // The chart/count query plan for however many sources are selected: one + // source keeps its severity-grouped histogram, several get one count() + // series each (stacked by source). + const histogramSpecs = useMemo(() => { + if (isMultiSource) return multiHistogramSpecs; + if (searchedSource == null || histogramTimeChartConfig == null) return []; + return [{ source: searchedSource, config: histogramTimeChartConfig }]; + }, [ + isMultiSource, + multiHistogramSpecs, + searchedSource, + histogramTimeChartConfig, + ]); + const onFormSubmit = useCallback>( e => { e.preventDefault(); @@ -2231,6 +2289,22 @@ export function DBSearchPage() { }; }, [chartConfig, searchedTimeRange, aliasWith]); + // The sidebar reads facets, values, and pins across everything selected; + // with one source that is exactly the single-source sidebar. + const filterSidebarSources = useMemo(() => { + if (isMultiSource) { + // Facet queries want each source's search shape (its own FROM, + // connection, and WHERE), not the aggregated histogram config. + return searchStreamSpecs.map(({ source, config }) => ({ + source, + config: { ...config, orderBy: undefined }, + })); + } + return searchedSource != null + ? [{ source: searchedSource, config: filtersChartConfig }] + : []; + }, [isMultiSource, searchStreamSpecs, searchedSource, filtersChartConfig]); + const openNewSourceModal = useCallback(() => { setNewSourceModalOpened(true); }, []); @@ -2789,7 +2863,7 @@ export function DBSearchPage() { height: '100%', }} > - {!isFilterSidebarCollapsed && !isMultiSource && ( + {!isFilterSidebarCollapsed && ( setIsFilterSidebarCollapsed(false) } - histogramTimeChartConfig={histogramTimeChartConfig} + histogramSpecs={histogramSpecs} /> - ) : ( <> - {/* The histogram, total count, and filters sidebar come - with the next change; searching several sources - returns the merged results table on its own. */} + + + + {isFilterSidebarCollapsed && ( + + setIsFilterSidebarCollapsed(false) + } + /> + )} + + + {shouldShowLiveModeHint && ( + + )} + + + + + setIsFilterSidebarCollapsed(false) } - histogramTimeChartConfig={histogramTimeChartConfig} + histogramSpecs={histogramSpecs} enableParallelQueries /> @@ -2990,14 +3095,9 @@ export function DBSearchPage() { className={searchPageStyles.timeChartContainer} mih="0" > - () =>
); // Multi-source components pull in DBRowSidePanel (and its deep import graph), // which this test isolates away just like DBSqlRowTableWithSidebar above. jest.mock('../components/SearchResultsTable', () => () =>
); +jest.mock('../components/SearchHistogram', () => ({ + SearchHistogram: () =>
, + SearchTotalCount: () =>
, +})); jest.mock('../components/PatternTable', () => () =>
); jest.mock('../components/Search/DBSearchHeatmapChart', () => ({ DBSearchHeatmapChart: () =>
, diff --git a/packages/app/src/components/DBSearchPageFilters.tsx b/packages/app/src/components/DBSearchPageFilters.tsx index 67a154aae7..55a60c41f7 100644 --- a/packages/app/src/components/DBSearchPageFilters.tsx +++ b/packages/app/src/components/DBSearchPageFilters.tsx @@ -8,6 +8,7 @@ import { FilterState } from '@hyperdx/common-utils/dist/filters'; import { BuilderChartConfigWithDateRange, SourceKind, + TSource, } from '@hyperdx/common-utils/dist/types'; import { Accordion, @@ -51,22 +52,23 @@ import { import { IS_CLICKHOUSE_BUILD } from '@/config'; import { useColumns, - useGetValuesDistribution, useJsonColumns, + useMergedValuesDistribution, useTableMetadata, } from '@/hooks/useMetadata'; +import { useMultiSourceColumns } from '@/hooks/useMultiSourceSearch'; import useResizable from '@/hooks/useResizable'; import { usePinnedFiltersApi } from '@/pinnedFilters'; import { FilterStateHook, IS_ROOT_SPAN_COLUMN_NAME, - usePinnedFilters, + usePinnedFiltersForSources, } from '@/searchFilters'; import { useSource } from '@/source'; import { useLocalStorage } from '@/utils'; import { FilterSettingsPanel } from './DBSearchPageFilters/FilterSettingsPopover'; -import { useFetchFacets } from './DBSearchPageFilters/hooks'; +import { useFetchFacetsForSources } from './DBSearchPageFilters/hooks'; import { NestedFilterGroup } from './DBSearchPageFilters/NestedFilterGroup'; import { PinShareIndicator, @@ -82,6 +84,17 @@ import { import resizeStyles from '@styles/ResizablePanel.module.scss'; import classes from '@styles/SearchPage.module.scss'; +// Placeholder used when no source is selected yet; nothing queries with it. +const EMPTY_FILTER_CHART_CONFIG = { + connection: '', + from: { databaseName: '', tableName: '' }, + timestampValueExpression: '', + select: '', + where: '', + whereLanguage: 'sql' as const, + dateRange: [new Date(0), new Date(0)] as [Date, Date], +}; + /* The initial number of values per filter to render */ const INITIAL_MAX_VALUES_DISPLAYED = 10; @@ -402,7 +415,8 @@ export type FilterGroupProps = { isDefaultExpanded?: boolean; showFilterCounts?: boolean; 'data-testid'?: string; - chartConfig: BuilderChartConfigWithDateRange; + /** One config per selected source; value counts are summed across them. */ + chartConfigs: BuilderChartConfigWithDateRange[]; isLive?: boolean; onRangeChange?: (range: { min: number; max: number }) => void; distributionKey?: string; @@ -428,7 +442,7 @@ const FilterGroupBody = ({ onLoadMore, loadMoreLoading, hasLoadedMore, - chartConfig, + chartConfigs, isLive, distributionKey, showDistributions, @@ -449,7 +463,7 @@ const FilterGroupBody = ({ onLoadMore: (key: string) => void; loadMoreLoading: boolean; hasLoadedMore: boolean; - chartConfig: BuilderChartConfigWithDateRange; + chartConfigs: BuilderChartConfigWithDateRange[]; isLive?: boolean; distributionKey?: string; showDistributions: boolean; @@ -463,16 +477,19 @@ const FilterGroupBody = ({ const [recentlyMoved, setRecentlyMoved] = useState>( new Set(), ); - // For live searches, don't refresh percentages when date range changes + // For live searches, don't refresh percentages when date range changes. + // Every source's config carries the same searched range, so the first one + // speaks for all of them. + const primaryDateRange = chartConfigs[0]?.dateRange; const [dateRange, setDateRange] = useState<[Date, Date]>( - chartConfig.dateRange, + primaryDateRange ?? EMPTY_FILTER_CHART_CONFIG.dateRange, ); useEffect(() => { - if (!isLive) { - setDateRange(chartConfig.dateRange); + if (!isLive && primaryDateRange != null) { + setDateRange(primaryDateRange); } - }, [chartConfig.dateRange, isLive]); + }, [primaryDateRange, isLive]); const handleSetSearch = useCallback( (value: string) => { @@ -484,25 +501,23 @@ const FilterGroupBody = ({ [hasLoadedMore, name, onLoadMore], ); + const distributionConfigs = useMemo( + () => chartConfigs.map(config => ({ ...config, dateRange })), + [chartConfigs, dateRange], + ); const { data: distributionData, isFetching: isFetchingDistribution, error: distributionError, - } = useGetValuesDistribution( + } = useMergedValuesDistribution( { - chartConfig: { ...chartConfig, dateRange }, + chartConfigs: distributionConfigs, key: distributionKey || name, limit: 100, // The 100 most common values are enough to find any values that are present in at least 1% of rows }, - { - enabled: showDistributions, - }, + { enabled: showDistributions }, ); - useEffect(() => { - onFetchingDistributionChange(isFetchingDistribution); - }, [isFetchingDistribution, onFetchingDistributionChange]); - useEffect(() => { if (distributionError) { notifications.show({ @@ -515,6 +530,10 @@ const FilterGroupBody = ({ } }, [distributionError, onDistributionError]); + useEffect(() => { + onFetchingDistributionChange(isFetchingDistribution); + }, [isFetchingDistribution, onFetchingDistributionChange]); + const totalAppliedFiltersSize = selectedValues.included.size + selectedValues.excluded.size + @@ -900,7 +919,7 @@ export const FilterGroup = ({ isDefaultExpanded, showFilterCounts, 'data-testid': dataTestId, - chartConfig, + chartConfigs, isLive, distributionKey, onRangeChange, @@ -1040,7 +1059,7 @@ export const FilterGroup = ({ onLoadMore={onLoadMore} loadMoreLoading={loadMoreLoading} hasLoadedMore={hasLoadedMore} - chartConfig={chartConfig} + chartConfigs={chartConfigs} isLive={isLive} distributionKey={distributionKey} showDistributions={showDistributions} @@ -1061,10 +1080,9 @@ const DBSearchPageFiltersComponent = ({ clearFilter, setFilterValue: _setFilterValue, isLive, - chartConfig, + sources, analysisMode, setAnalysisMode, - sourceId, showDelta, denoiseResults, setDenoiseResults, @@ -1076,8 +1094,11 @@ const DBSearchPageFiltersComponent = ({ analysisMode: 'results' | 'delta' | 'pattern'; setAnalysisMode: (mode: 'results' | 'delta' | 'pattern') => void; isLive: boolean; - chartConfig: BuilderChartConfigWithDateRange; - sourceId?: string; + /** + * One entry per selected source. Facets, values, and pins merge across all + * of them; a single source behaves exactly as before. + */ + sources: { source: TSource; config: BuilderChartConfigWithDateRange }[]; showDelta: boolean; denoiseResults: boolean; setDenoiseResults: (denoiseResults: boolean) => void; @@ -1096,6 +1117,23 @@ const DBSearchPageFiltersComponent = ({ }, [_setFilterValue], ); + // The first selected source is the "primary": it anchors the things that + // are inherently about one table (schema preview, the analysis-mode tabs). + // Everything users read or click in the list itself merges across sources. + const primarySource = sources[0]?.source; + const sourceId = primarySource?.id; + const chartConfig = sources[0]?.config ?? EMPTY_FILTER_CHART_CONFIG; + const sourceIds = useMemo(() => sources.map(s => s.source.id), [sources]); + const chartConfigs = useMemo(() => sources.map(s => s.config), [sources]); + const facetSpecs = useMemo( + () => + sources.map(({ source, config }) => ({ + sourceId: source.id, + chartConfig: config, + })), + [sources], + ); + const { toggleFilterPin, toggleFieldPin, @@ -1111,7 +1149,7 @@ const DBSearchPageFiltersComponent = ({ resetSharedFilters, hasPersonalPins, hasSharedPins, - } = usePinnedFilters(sourceId ?? null); + } = usePinnedFiltersForSources(sourceIds); const { data: pinnedFiltersApiData } = usePinnedFiltersApi(sourceId ?? null); const [isSharedFiltersVisible, setSharedFiltersVisible] = useLocalStorage( 'hdx-shared-filters-visible', @@ -1144,6 +1182,11 @@ const DBSearchPageFiltersComponent = ({ chartConfig.dateRange, ); + // Filter keys can come from any selected source's schema, so escaping is + // resolved against the union of their columns. + const { columnsBySourceId } = useMultiSourceColumns( + useMemo(() => sources.map(s => s.source), [sources]), + ); const { data: columns } = useColumns({ databaseName: chartConfig.from.databaseName, tableName: chartConfig.from.tableName, @@ -1164,10 +1207,13 @@ const DBSearchPageFiltersComponent = ({ // Conditionally backtick-quote facet keys that contain special characters and // match known column names, so they can be used in the ClickHouse query to get // key values. - const knownColumns = useMemo( - () => (columns ? new Set(columns.map(c => c.name)) : new Set()), - [columns], - ); + const knownColumns = useMemo(() => { + const names = new Set(columns?.map(c => c.name) ?? []); + for (const sourceColumns of columnsBySourceId.values()) { + for (const name of sourceColumns) names.add(name); + } + return names; + }, [columns, columnsBySourceId]); const [showMoreFields, setShowMoreFields] = useState(false); const { @@ -1178,9 +1224,8 @@ const DBSearchPageFiltersComponent = ({ loadMoreFacetsForKey, loadMoreLoadingKeys, extraFacetKeys, - } = useFetchFacets({ - chartConfig, - sourceId: sourceId ?? null, + } = useFetchFacetsForSources({ + specs: facetSpecs, dateRange, mode: showAllValues ? 'all' : 'exact', filterState, @@ -1516,7 +1561,7 @@ const DBSearchPageFiltersComponent = ({ ); }) } - chartConfig={chartConfig} + chartConfigs={chartConfigs} isLive={isLive} /> ))} @@ -1571,7 +1616,7 @@ const DBSearchPageFiltersComponent = ({ entry.range != null))) ); })()} - chartConfig={chartConfig} + chartConfigs={chartConfigs} isLive={isLive} onRangeChange={range => setFilterRange(facet.key, range)} /> @@ -1598,7 +1643,7 @@ const DBSearchPageFiltersComponent = ({ loadMoreLoadingKeys, showFilterCounts, isFacetsLoading, - chartConfig, + chartConfigs, isLive, setFilterRange, tableMetadata, diff --git a/packages/app/src/components/DBSearchPageFilters/NestedFilterGroup.tsx b/packages/app/src/components/DBSearchPageFilters/NestedFilterGroup.tsx index 4145d8e917..6291a6136c 100644 --- a/packages/app/src/components/DBSearchPageFilters/NestedFilterGroup.tsx +++ b/packages/app/src/components/DBSearchPageFilters/NestedFilterGroup.tsx @@ -39,7 +39,8 @@ type NestedFilterGroupProps = { hasLoadedMore: Record; isDefaultExpanded?: boolean; 'data-testid'?: string; - chartConfig: any; // Using any to avoid importing ChartConfigWithDateRange + /** One config per selected source; counts are summed across them. */ + chartConfigs: any[]; // `any` avoids importing ChartConfigWithDateRange isLive?: boolean; }; @@ -70,7 +71,7 @@ export const NestedFilterGroup = ({ hasLoadedMore, isDefaultExpanded, 'data-testid': dataTestId, - chartConfig, + chartConfigs, isLive, }: NestedFilterGroupProps) => { const selectedValues: FilterState = useMemo( @@ -253,7 +254,7 @@ export const NestedFilterGroup = ({ hasLoadedMore={hasLoadedMore[child.key] || false} showFilterCounts={showFilterCounts} isDefaultExpanded={childHasSelections} - chartConfig={chartConfig} + chartConfigs={chartConfigs} isLive={isLive} />
diff --git a/packages/app/src/components/DBSearchPageFilters/hooks.ts b/packages/app/src/components/DBSearchPageFilters/hooks.ts index a9d676102a..d9dd4319e6 100644 --- a/packages/app/src/components/DBSearchPageFilters/hooks.ts +++ b/packages/app/src/components/DBSearchPageFilters/hooks.ts @@ -17,6 +17,7 @@ import { useMapColumns, useMetadataWithSettings, } from '@/hooks/useMetadata'; +import { useMultiSourceSlots } from '@/hooks/useSourceSlots'; import { escapeFilterStateKeys, usePinnedFilters } from '@/searchFilters'; import { useSource } from '@/source'; import { mergePath } from '@/utils'; @@ -372,3 +373,147 @@ export function useFetchFacets({ extraFacetKeys, }; } + +export type SourceFacetSpec = { + sourceId: string; + chartConfig: BuilderChartConfigWithDateRange; +}; + +/** Slot hook: the full facet pipeline for one selected source. */ +function useSourceFacetsSlot( + spec: SourceFacetSpec | undefined, + opts: { + dateRange: [Date, Date]; + mode: 'all' | 'exact'; + filterState?: FilterState; + showMoreFields?: boolean; + }, +) { + const query = useFetchFacets({ + chartConfig: spec?.chartConfig ?? STUB_FACET_CONFIG, + sourceId: spec?.sourceId ?? null, + dateRange: opts.dateRange, + mode: opts.mode, + filterState: opts.filterState, + showMoreFields: opts.showMoreFields, + enabled: spec != null, + }); + return query; +} + +const STUB_FACET_CONFIG: BuilderChartConfigWithDateRange = { + connection: '', + from: { databaseName: '', tableName: '' }, + timestampValueExpression: '', + select: '', + where: '', + whereLanguage: 'sql', + dateRange: [new Date(0), new Date(0)], +}; + +/** + * Facets across every selected source: fields and values merged by field + * path, values unioned in first-seen order. "Load more" fans out to each + * source and unions what comes back, so a high-cardinality field expands + * across the whole search rather than one table. + * + * With a single source this is `useFetchFacets` for that source, unchanged. + */ +export function useFetchFacetsForSources({ + specs, + dateRange, + mode, + filterState, + showMoreFields, +}: { + specs: SourceFacetSpec[]; + dateRange: [Date, Date]; + mode: 'all' | 'exact'; + filterState?: FilterState; + showMoreFields?: boolean; +}) { + const slots = useMultiSourceSlots(specs, useSourceFacetsSlot, { + dateRange, + mode, + filterState, + showMoreFields, + }); + + const merged = useMemo(() => { + const byKey = new Map< + string, + { values: (string | boolean)[]; seen: Set } + >(); + let sawAny = false; + for (const slot of slots) { + const facets = slot.data.keyValues; + if (facets == null) continue; + sawAny = true; + for (const facet of facets) { + let entry = byKey.get(facet.key); + if (entry == null) { + entry = { values: [], seen: new Set() }; + byKey.set(facet.key, entry); + } + for (const value of facet.value) { + if (!entry.seen.has(value)) { + entry.seen.add(value); + entry.values.push(value); + } + } + } + } + const keyValues = sawAny + ? [...byKey.entries()].map(([key, entry]) => ({ + key, + value: entry.values, + })) + : undefined; + + const keys = slots.flatMap(slot => slot.data.keys ?? []); + const seenPaths = new Set(); + const mergedKeys = keys.filter(field => { + const id = `${field.path.join('.')}|${field.type}`; + if (seenPaths.has(id)) return false; + seenPaths.add(id); + return true; + }); + + return { keys: mergedKeys.length > 0 ? mergedKeys : undefined, keyValues }; + }, [slots]); + + const loadMoreFacetsForKey = useCallback( + async (key: string) => { + await Promise.all(slots.map(slot => slot.loadMoreFacetsForKey(key))); + }, + [slots], + ); + + const loadMoreLoadingKeys = useMemo(() => { + const keys = new Set(); + for (const slot of slots) { + for (const key of slot.loadMoreLoadingKeys) keys.add(key); + } + return keys; + }, [slots]); + + const extraFacetKeys = useMemo(() => { + const keys = new Set(); + for (const slot of slots) { + for (const key of slot.extraFacetKeys) keys.add(key); + } + return keys; + }, [slots]); + + return { + data: merged, + isLoading: slots.some(s => s.isLoading), + isFetching: slots.some(s => s.isFetching), + // A single failing source shouldn't blank the sidebar; surface the first. + error: slots.find(s => s.error != null)?.error, + loadMoreFacetsForKey, + loadMoreLoadingKeys, + extraFacetKeys, + areExtraFacetsLoading: slots.some(s => s.areExtraFacetsLoading), + }; +} diff --git a/packages/app/src/components/SearchHistogram.tsx b/packages/app/src/components/SearchHistogram.tsx new file mode 100644 index 0000000000..55dbaf1481 --- /dev/null +++ b/packages/app/src/components/SearchHistogram.tsx @@ -0,0 +1,421 @@ +import { useMemo, useState } from 'react'; +import { + ColumnMetaType, + filterColumnMetaByType, + JSDataType, + ResponseJSON, +} from '@hyperdx/common-utils/dist/clickhouse'; +import { + BuilderChartConfigWithDateRange, + DisplayType, + TSource, +} from '@hyperdx/common-utils/dist/types'; +import { Text } from '@mantine/core'; +import { keepPreviousData } from '@tanstack/react-query'; + +import api from '@/api'; +import { + convertToTimeChartConfig, + formatResponseForTimeChart, + useTimeChartSettings, +} from '@/ChartUtils'; +import ChartContainer from '@/components/charts/ChartContainer'; +import ChartErrorState from '@/components/charts/ChartErrorState'; +import { type ActiveClickPayload, MemoChart } from '@/HDXMultiSeriesTimeChart'; +import { useQueriedChartConfig } from '@/hooks/useChartConfig'; +import { useMultiSourceSlots } from '@/hooks/useSourceSlots'; +import type { NumberFormat } from '@/types'; + +import { DBTimeChart, type SeriesGroupFilter } from './DBTimeChart'; +import { getMultiSourceColor } from './MultiSourceBadge'; +import SearchTotalCountChart from './SearchTotalCountChart'; + +/** Synthetic group column tagged onto each source's histogram rows. */ +const SOURCE_GROUP_COLUMN = '__hdx_source'; + +export type SearchHistogramSpec = { + source: TSource; + /** Per-source count() histogram config (canonical WHERE, no groupBy). */ + config: BuilderChartConfigWithDateRange; + /** When set, the source doesn't run (mirrors MultiSourceStreamSpec). */ + disabledReason?: string; +}; + +// Placeholder for unused hook slots; never queried (enabled: false). +const STUB_CONFIG: BuilderChartConfigWithDateRange = { + connection: '', + from: { databaseName: '', tableName: '' }, + timestampValueExpression: '', + select: '', + where: '', + whereLanguage: 'sql', + dateRange: [new Date(0), new Date(0)], +}; + +type HistogramSlotState = { + data: ReturnType['data']; + isLoading: boolean; + isError: boolean; + error: Error | null; +}; + +function useHistogramSlot( + spec: SearchHistogramSpec | undefined, + { + enabled, + queryKeyPrefix, + enableParallelQueries, + parallelizeWhenPossible, + }: { + enabled: boolean; + queryKeyPrefix: string; + enableParallelQueries?: boolean; + parallelizeWhenPossible?: boolean; + }, +): HistogramSlotState { + const queriedConfig = useMemo( + () => convertToTimeChartConfig(spec?.config ?? STUB_CONFIG), + [spec?.config], + ); + + const { data, isLoading, isError, error } = useQueriedChartConfig( + queriedConfig, + { + // Key shape mirrors DBTimeChart/SearchTotalCountChart so TanStack can + // de-dupe the histogram and total-count consumers of the same source. + queryKey: [ + queryKeyPrefix, + queriedConfig, + 'chunked', + { + disableQueryChunking: false, + enableParallelQueries, + parallelizeWhenPossible, + }, + ], + placeholderData: keepPreviousData, + enableQueryChunking: true, + enableParallelQueries: enableParallelQueries && parallelizeWhenPossible, + enabled: enabled && spec != null && spec.disabledReason == null, + }, + ); + + // Stable identity per content change so the slots array (and everything + // memoized on it) doesn't churn on unrelated renders. + return useMemo( + () => ({ data, isLoading, isError, error: error ?? null }), + [data, isLoading, isError, error], + ); +} + +/** + * Runs one count() histogram query per selected source (one hook slot per + * source, see useMultiSourceSlots) and merges the responses into a single + * response shape with a synthetic source-name group column — so the standard + * time-chart transform naturally yields one series per source. + */ +function useMultiSourceHistogram( + specs: SearchHistogramSpec[], + { + enabled = true, + queryKeyPrefix, + enableParallelQueries, + }: { + enabled?: boolean; + queryKeyPrefix: string; + enableParallelQueries?: boolean; + }, +) { + const { data: me, isLoading: isLoadingMe } = api.useMe(); + const slots = useMultiSourceSlots(specs, useHistogramSlot, { + enabled: enabled && !isLoadingMe, + queryKeyPrefix, + enableParallelQueries, + parallelizeWhenPossible: me?.team?.parallelizeWhenPossible, + }); + + const isLoading = slots.some(s => s.isLoading); + const allFailed = slots.length > 0 && slots.every(s => s.isError); + const anyError = slots.some(s => s.isError); + const error = slots.find(s => s.error != null)?.error ?? undefined; + const isComplete = + slots.length > 0 && slots.every(s => s.isError || !!s.data?.isComplete); + + const mergedResponse: ResponseJSON> | undefined = + useMemo(() => { + let meta: ColumnMetaType[] | undefined; + const data: Record[] = []; + for (let i = 0; i < specs.length; i++) { + const response = slots[i]?.data; + if (response?.meta == null || response.meta.length === 0) continue; + if (meta == null) { + meta = [ + ...response.meta, + { name: SOURCE_GROUP_COLUMN, type: 'String' }, + ]; + } + const sourceName = specs[i].source.name; + for (const row of response.data ?? []) { + data.push({ ...row, [SOURCE_GROUP_COLUMN]: sourceName }); + } + } + return meta ? { data, meta, rows: data.length } : undefined; + }, [slots, specs]); + + return { mergedResponse, isLoading, allFailed, anyError, error, isComplete }; +} + +const EMPTY_NUMBER_FORMATS = new Map(); + +/** + * The multi-source search histogram: one stacked count() series per selected + * source, colored consistently with the results-table badges. A thin + * counterpart to DBTimeChart — drag-to-zoom and the legend work; per-series + * drill-down/pinned tooltips are single-source features and are omitted. + */ +function MergedSourcesTimeChart({ + specs, + enabled = true, + queryKeyPrefix, + enableParallelQueries, + onTimeRangeSelect, + showLegend = true, +}: { + specs: SearchHistogramSpec[]; + enabled?: boolean; + queryKeyPrefix: string; + enableParallelQueries?: boolean; + onTimeRangeSelect?: (start: Date, end: Date) => void; + showLegend?: boolean; +}) { + const { mergedResponse, isLoading, allFailed, error, isComplete } = + useMultiSourceHistogram(specs, { + enabled, + queryKeyPrefix, + enableParallelQueries, + }); + + const firstConfig = specs[0]?.config; + const { dateRange, granularity } = useTimeChartSettings( + firstConfig ?? STUB_CONFIG, + ); + + const [activeClickPayload, setActiveClickPayload] = useState< + ActiveClickPayload | undefined + >(); + + const colorBySourceName = useMemo( + () => + new Map( + specs.map((spec, i) => [spec.source.name, getMultiSourceColor(i)]), + ), + [specs], + ); + + const formatted = useMemo(() => { + if (mergedResponse == null) { + return null; + } + try { + const result = formatResponseForTimeChart({ + currentPeriodResponse: mergedResponse, + dateRange, + granularity, + generateEmptyBuckets: true, + }); + // One series per source: recolor to the shared per-source palette so + // the histogram matches the table badges and status chips. + for (const line of result.lineData) { + const color = colorBySourceName.get(line.dataKey); + if (color != null) { + line.color = color; + } + } + return result; + } catch (e) { + console.error(e); + return null; + } + }, [mergedResponse, dateRange, granularity, colorBySourceName]); + + if (allFailed && error) { + return ; + } + + return ( + + {isLoading && formatted == null ? ( +
+ Loading Chart Data... +
+ ) : formatted == null || formatted.graphResults.length === 0 ? ( +
+ No data found within time range. +
+ ) : ( + + )} + + ); +} + +/** + * Summed "N Results" across every selected source, sharing the histogram's + * per-source queries (identical query keys) so it adds no ClickHouse load. + */ +function MergedSourcesTotalCount({ + specs, + enabled = true, + queryKeyPrefix, + enableParallelQueries, +}: { + specs: SearchHistogramSpec[]; + enabled?: boolean; + queryKeyPrefix: string; + enableParallelQueries?: boolean; +}) { + const { mergedResponse, isLoading, allFailed } = useMultiSourceHistogram( + specs, + { + enabled, + queryKeyPrefix, + enableParallelQueries, + }, + ); + + const totalCount = useMemo(() => { + if (mergedResponse == null) return undefined; + // The count column may be renamed (e.g. via materialized views); fall back + // to the first numeric column, mirroring SearchTotalCountChart. + const countColumn = + mergedResponse.meta?.find(c => c.name === 'count()')?.name ?? + filterColumnMetaByType(mergedResponse.meta ?? [], [ + JSDataType.Number, + ])?.[0]?.name ?? + 'count()'; + return mergedResponse.data.reduce( + (sum: number, row: any) => sum + (Number.parseInt(row[countColumn]) || 0), + 0, + ); + }, [mergedResponse]); + + return ( + + {isLoading && totalCount == null ? ( + ··· Results + ) : totalCount != null && !allFailed ? ( + `${totalCount.toLocaleString()} Results` + ) : ( + '0 Results' + )} + + ); +} + +/** + * The search page's histogram, for any number of selected sources. + * + * One source keeps the full-featured DBTimeChart — series drill-down, focus, + * the pinned tooltip, MV optimization — grouped by severity/status, which is + * what a single source's chart has always shown. Several sources can't share + * a severity vocabulary, so they stack one count() series per source instead, + * and the merged chart trades the per-series drill-down for that. + */ +export function SearchHistogram({ + specs, + enabled = true, + queryKeyPrefix, + enableParallelQueries, + onTimeRangeSelect, + onFocusSeries, + showLegend, +}: { + /** One spec per selected source; N=1 is the single-source histogram. */ + specs: SearchHistogramSpec[]; + enabled?: boolean; + queryKeyPrefix: string; + enableParallelQueries?: boolean; + onTimeRangeSelect?: (start: Date, end: Date) => void; + /** Focus a severity/status series into the search (single source only). */ + onFocusSeries?: (filters: SeriesGroupFilter[]) => void; + showLegend?: boolean; +}) { + if (specs.length === 1) { + return ( + + ); + } + + return ( + + ); +} + +/** + * "N Results" for any number of sources: the single-source count query, or the + * sum across sources. Shares the histogram's per-source queries either way, so + * it adds no ClickHouse load. + */ +export function SearchTotalCount({ + specs, + enabled = true, + queryKeyPrefix, + enableParallelQueries, +}: { + specs: SearchHistogramSpec[]; + enabled?: boolean; + queryKeyPrefix: string; + enableParallelQueries?: boolean; +}) { + if (specs.length === 1) { + return ( + + ); + } + + return ( + + ); +} diff --git a/packages/app/src/components/__tests__/DBSearchPageFilters.test.tsx b/packages/app/src/components/__tests__/DBSearchPageFilters.test.tsx index 0353c68d4f..a1d4359c93 100644 --- a/packages/app/src/components/__tests__/DBSearchPageFilters.test.tsx +++ b/packages/app/src/components/__tests__/DBSearchPageFilters.test.tsx @@ -1,4 +1,3 @@ -import { UseQueryResult } from '@tanstack/react-query'; import { screen, within } from '@testing-library/react'; import userEvent from '@testing-library/user-event'; @@ -13,7 +12,7 @@ import { groupFacetsByBaseName, parseMapFieldName, } from '@/components/DBSearchPageFilters/utils'; -import { useGetValuesDistribution } from '@/hooks/useMetadata'; +import { useMergedValuesDistribution } from '@/hooks/useMetadata'; describe('cleanClickHouseExpression', () => { it('should remove toString wrapper', () => { @@ -50,9 +49,9 @@ describe('cleanClickHouseExpression', () => { }); jest.mock('@/hooks/useMetadata', () => ({ - useGetValuesDistribution: jest + useMergedValuesDistribution: jest .fn() - .mockReturnValue({ data: undefined, isFetching: false, error: undefined }), + .mockReturnValue({ data: undefined, isFetching: false, error: null }), })); describe('cleanedFacetName', () => { @@ -396,18 +395,20 @@ describe('FilterGroup', () => { loadMoreLoading: false, hasLoadedMore: false, isDefaultExpanded: true, - chartConfig: { - from: { - databaseName: 'test_db', - tableName: 'test_table', + chartConfigs: [ + { + from: { + databaseName: 'test_db', + tableName: 'test_table', + }, + select: '', + where: '', + whereLanguage: 'sql', + timestampValueExpression: '', + connection: 'test_connection', + dateRange: [new Date('2024-01-01'), new Date('2024-01-02')], }, - select: '', - where: '', - whereLanguage: 'sql', - timestampValueExpression: '', - connection: 'test_connection', - dateRange: [new Date('2024-01-01'), new Date('2024-01-02')], - }, + ], }; it('should sort options alphabetically by default', () => { @@ -441,7 +442,7 @@ describe('FilterGroup', () => { }); it('should show selected items first, then sort by counts, if percentages when they are enabled', () => { - jest.mocked(useGetValuesDistribution).mockReturnValue({ + jest.mocked(useMergedValuesDistribution).mockReturnValue({ data: new Map([ ['apple', 30], ['banana', 20], @@ -449,7 +450,7 @@ describe('FilterGroup', () => { ]), isFetching: false, error: null, - } as UseQueryResult>); + }); renderWithMantine( { }); it('should show percentages, if enabled', async () => { - jest.mocked(useGetValuesDistribution).mockReturnValue({ + jest.mocked(useMergedValuesDistribution).mockReturnValue({ data: new Map([ ['apple', 99.2], ['zebra', 0.6], ]), isFetching: false, error: null, - } as UseQueryResult>); + }); renderWithMantine( { onLoadMore: jest.fn(), loadMoreLoading: {} as Record, hasLoadedMore: {} as Record, - chartConfig: { - from: { databaseName: 'test_db', tableName: 'test_table' }, - select: '', - where: '', - whereLanguage: 'sql', - timestampValueExpression: '', - connection: 'test_connection', - dateRange: [new Date('2024-01-01'), new Date('2024-01-02')], - }, + chartConfigs: [ + { + from: { databaseName: 'test_db', tableName: 'test_table' }, + select: '', + where: '', + whereLanguage: 'sql', + timestampValueExpression: '', + connection: 'test_connection', + dateRange: [new Date('2024-01-01'), new Date('2024-01-02')], + }, + ], }; it('should not render child FilterGroups when collapsed', () => { diff --git a/packages/app/src/hooks/useMetadata.tsx b/packages/app/src/hooks/useMetadata.tsx index fca1559d5d..7e9acfda89 100644 --- a/packages/app/src/hooks/useMetadata.tsx +++ b/packages/app/src/hooks/useMetadata.tsx @@ -32,6 +32,7 @@ import api from '@/api'; import { IS_LOCAL_MODE } from '@/config'; import { LOCAL_STORE_CONNECTIONS_KEY } from '@/connection'; import { DEFAULT_FILTER_KEYS_FETCH_LIMIT } from '@/defaults'; +import { useMultiSourceSlots } from '@/hooks/useSourceSlots'; import { getMetadata } from '@/metadata'; import { useSource, useSources } from '@/source'; import { toArray } from '@/utils'; @@ -449,7 +450,7 @@ export function useMultipleGetKeyValues( }; } -export function useGetValuesDistribution( +function useGetValuesDistribution( { chartConfig, key, @@ -484,6 +485,79 @@ export function useGetValuesDistribution( }); } +/** Slot hook: value→count distribution for one source's config. */ +function useValuesDistributionSlot( + chartConfig: BuilderChartConfigWithDateRange | undefined, + { key, limit, enabled }: { key: string; limit: number; enabled: boolean }, +) { + const { data, isFetching, error } = useGetValuesDistribution( + { + chartConfig: chartConfig ?? STUB_DISTRIBUTION_CONFIG, + key, + limit, + }, + { enabled: enabled && chartConfig != null }, + ); + return useMemo( + () => ({ data, isFetching, error }), + [data, isFetching, error], + ); +} + +const STUB_DISTRIBUTION_CONFIG: BuilderChartConfigWithDateRange = { + connection: '', + from: { databaseName: '', tableName: '' }, + timestampValueExpression: '', + select: '', + where: '', + whereLanguage: 'sql', + dateRange: [new Date(0), new Date(0)], +}; + +/** + * Value→count distribution for a filter key across every selected source, + * summed. With one source this is exactly `useGetValuesDistribution`; with + * several, a value's count is the total across the sources that have it, so + * the sidebar's percentages describe the whole search rather than one table. + */ +export function useMergedValuesDistribution( + { + chartConfigs, + key, + limit, + }: { + chartConfigs: BuilderChartConfigWithDateRange[]; + key: string; + limit: number; + }, + options?: { enabled?: boolean }, +) { + const slots = useMultiSourceSlots(chartConfigs, useValuesDistributionSlot, { + key, + limit, + enabled: options?.enabled ?? true, + }); + + return useMemo(() => { + const merged = new Map(); + let any = false; + for (const slot of slots) { + if (slot.data == null) continue; + any = true; + for (const [value, count] of slot.data) { + merged.set(value, (merged.get(value) ?? 0) + count); + } + } + return { + data: any ? merged : undefined, + isFetching: slots.some(s => s.isFetching), + // Surface the first failure so the group can drop percentages rather + // than showing counts that silently omit a source. + error: slots.find(s => s.error != null)?.error ?? null, + }; + }, [slots]); +} + export function useGetKeyValues( { chartConfig, diff --git a/packages/app/src/searchFilters.tsx b/packages/app/src/searchFilters.tsx index ac193b7f26..5492f16a71 100644 --- a/packages/app/src/searchFilters.tsx +++ b/packages/app/src/searchFilters.tsx @@ -11,6 +11,7 @@ import { cleanClickHouseExpression, toQuotedClickHouseKeyExpression, } from './components/DBSearchPageFilters/utils'; +import { useMultiSourceSlots } from './hooks/useSourceSlots'; import { usePinnedFiltersApi, useUpdatePinnedFilters } from './pinnedFilters'; import { useLocalStorage } from './utils'; @@ -658,3 +659,102 @@ export function usePinnedFilters(sourceId: string | null) { hasSharedPins, }; } + +/** + * Pinned filters across every selected source. + * + * Pins are stored per source (personal pins in localStorage, shared pins in + * Mongo), so a search spanning several sources reads the union of their pins + * and writes to all of them: pinning `ServiceName` while searching logs and + * traces keeps it pinned in both, and unpinning clears it from both. With one + * source this is exactly `usePinnedFilters` for that source. + */ +export function usePinnedFiltersForSources(sourceIds: string[]) { + const slots = useMultiSourceSlots(sourceIds, usePinnedFiltersSlot, undefined); + const active = useMemo( + () => slots.slice(0, sourceIds.length), + [slots, sourceIds.length], + ); + + return useMemo(() => { + // Reads: a pin on any selected source shows in the sidebar. + const anyOf = + ( + pick: (s: PinnedFiltersHook) => (...a: A) => boolean, + ) => + (...args: A) => + active.some(slot => pick(slot)(...args)); + // Writes: fan out so a pin applies to the whole selection. Toggling is + // resolved against the merged view first, so a mixed state (pinned on one + // source, not another) resolves to "pin everywhere" rather than flipping + // each source independently. + const fanOut = + ( + pick: (s: PinnedFiltersHook) => (...a: A) => void, + isSet: (s: PinnedFiltersHook, ...a: A) => boolean, + ) => + (...args: A) => { + const shouldPin = !active.some(slot => isSet(slot, ...args)); + for (const slot of active) { + if (isSet(slot, ...args) !== shouldPin) { + pick(slot)(...args); + } + } + }; + + return { + pinnedFilters: active.reduce( + (acc, slot) => mergePinnedFilterValues(acc, slot.pinnedFilters), + {}, + ), + getPinnedFields: () => [ + ...new Set(active.flatMap(slot => slot.getPinnedFields())), + ], + isFilterPinned: anyOf(s => s.isFilterPinned), + isFieldPinned: anyOf(s => s.isFieldPinned), + isSharedFilterPinned: anyOf(s => s.isSharedFilterPinned), + isSharedFieldPinned: anyOf(s => s.isSharedFieldPinned), + toggleFilterPin: fanOut( + s => s.toggleFilterPin, + (s, property: string, value: string | boolean) => + s.isFilterPinned(property, value), + ), + toggleFieldPin: fanOut( + s => s.toggleFieldPin, + (s, key: string) => s.isFieldPinned(key), + ), + toggleSharedFilterPin: fanOut( + s => s.toggleSharedFilterPin, + (s, property: string, value: string | boolean) => + s.isSharedFilterPinned(property, value), + ), + toggleSharedFieldPin: fanOut( + s => s.toggleSharedFieldPin, + (s, key: string) => s.isSharedFieldPinned(key), + ), + resetPersonalPins: () => active.forEach(s => s.resetPersonalPins()), + resetSharedFilters: () => active.forEach(s => s.resetSharedFilters()), + hasPersonalPins: active.some(s => s.hasPersonalPins), + hasSharedPins: active.some(s => s.hasSharedPins), + }; + }, [active]); +} + +type PinnedFiltersHook = ReturnType; + +/** Slot hook: one source's pins. Unused slots pass null and stay inert. */ +function usePinnedFiltersSlot(sourceId: string | undefined) { + return usePinnedFilters(sourceId ?? null); +} + +/** Union two pinned-filter maps, deduping values per key. */ +function mergePinnedFilterValues( + a: PinnedFilters, + b: PinnedFilters, +): PinnedFilters { + const out: PinnedFilters = { ...a }; + for (const [key, values] of Object.entries(b)) { + out[key] = [...new Set([...(out[key] ?? []), ...values])]; + } + return out; +}