From 3113a0f5d7f2adf1cdf5bcec650de77a2c8cad70 Mon Sep 17 00:00:00 2001 From: Tom Alexander Date: Wed, 12 Aug 2026 13:01:42 -0400 Subject: [PATCH] refactor(app): extract the denoise pipeline out of the results table MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit DBSqlRowTable carried the whole denoise flow inline: mine patterns from a sample, find the ones covering more than the noise threshold, then filter the fetched rows against them. Three chained queries and their loading state, in the middle of a component that already does a lot. Pull it into useDenoisedRows plus the small summary that lists what was removed, and have the table call them. Same queries, same keys, same behavior — the component just stops owning it, and a second results table can reuse it rather than copy it. Also threads the last page's row count out of the paginated query, which a caller merging several sources needs to tell "this window is drained" from "this page stopped at its limit". --- packages/app/src/components/DBRowTable.tsx | 280 ++++++++++++------ .../app/src/components/MultiSourceBadge.tsx | 23 ++ .../app/src/hooks/useOffsetPaginatedQuery.tsx | 8 +- 3 files changed, 216 insertions(+), 95 deletions(-) create mode 100644 packages/app/src/components/MultiSourceBadge.tsx diff --git a/packages/app/src/components/DBRowTable.tsx b/packages/app/src/components/DBRowTable.tsx index 19574d849b..21834391a0 100644 --- a/packages/app/src/components/DBRowTable.tsx +++ b/packages/app/src/components/DBRowTable.tsx @@ -103,6 +103,7 @@ import { useLocalStorage, usePrevious, } from '@/utils'; +import { MULTI_SOURCE_ROW_FIELDS } from '@/utils/multiSourceMerge'; import ChartErrorState, { ChartErrorStateVariant, @@ -124,6 +125,7 @@ import { useExpandableRows, } from './ExpandableRowTable'; import LogLevel from './LogLevel'; +import { SourceBadge } from './MultiSourceBadge'; import styles from '@styles/LogTable.module.scss'; @@ -167,6 +169,7 @@ function getResolvedColumnSize( const jsType = opts.columnTypeMap.get(column)?._type; if (jsType === JSDataType.Date) return 170; if (column === opts.logLevelColumn) return 115; + if (column === MULTI_SOURCE_ROW_FIELDS.SOURCE_NAME) return 140; return 160; } @@ -615,6 +618,30 @@ export const RawLogTable = memo( const strValue = typeof value === 'string' ? value : `${value}`; + // Multi-source search tags each merged row with its origin + // source; render it as a colored badge (color assigned by the + // merge layer, consistent with the histogram series). + if (column === MULTI_SOURCE_ROW_FIELDS.SOURCE_NAME) { + return ( + + ); + } + + // Multi-source rows project NULL where a source lacks the + // field (e.g. Duration or a picked column for log rows); show a + // quiet dash instead of the literal "null". + if ( + value == null && + info.row.original[MULTI_SOURCE_ROW_FIELDS.SOURCE_ID] != null + ) { + return —; + } + if (column === logLevelColumn) { return ; } @@ -1476,6 +1503,15 @@ export function useConfigWithAdditionalSelect( }, [primaryKey, partitionKey, config, tableMetadata, columns, sourceId]); } +/** + * The user's SELECT columns, keyed by result-set name, with the row-identity + * columns the query appends (primary/partition/block keys) trimmed off. + * + * Positional rather than name-based because ClickHouse may rewrite column + * names, and the SELECT length can differ from the returned column count + * (e.g. `SELECT *`). Exported for SearchResultsTable, which resolves the same + * columns when rendering a single source's own SELECT. + */ function selectColumnMapWithoutAdditionalKeys( selectMeta: ColumnMetaType[] | undefined, additionalKeysLength: number | undefined, @@ -1503,6 +1539,144 @@ function selectColumnMapWithoutAdditionalKeys( export type DBRowTableVariant = 'default' | 'muted'; +/** + * Drop rows matching "noisy" event patterns (patterns covering more than + * DENOISE_NOISE_THRESHOLD of a sample) from an already-fetched row set. + * + * Extracted so the search results table and DBSqlRowTable share one + * implementation. Denoising is inherently single-source: it mines patterns + * from one table's body column against that source's severity expression. + */ +function useDenoisedRows({ + config, + sourceId, + processedRows, + patternColumn, + denoiseResults, + isLive, +}: { + config: BuilderChartConfigWithDateRange; + sourceId?: string; + processedRows: Record[]; + /** Result column the patterns are mined from (the last SELECT column). */ + patternColumn: string | undefined; + denoiseResults: boolean; + isLive?: boolean; +}) { + const { data: source } = useSource({ id: sourceId }); + const groupedPatterns = useGroupedPatterns({ + config, + samples: DENOISE_SAMPLE_SIZE, + bodyValueExpression: patternColumn ?? '', + severityTextExpression: + (source?.kind === SourceKind.Log + ? source.severityTextExpression + : undefined) ?? '', + totalCount: undefined, + enabled: denoiseResults, + }); + const noisyPatterns = useQuery({ + queryKey: ['noisy-patterns', config], + queryFn: async () => { + return Object.values(groupedPatterns.data).filter( + p => + p.count / (groupedPatterns.sampledRowCount ?? 1) > + DENOISE_NOISE_THRESHOLD, + ); + }, + enabled: + denoiseResults && + groupedPatterns.data != null && + Object.values(groupedPatterns.data).length > 0 && + groupedPatterns.miner != null, + }); + const noisyPatternIds = useMemo(() => { + return noisyPatterns.data?.map(p => p.id) ?? []; + }, [noisyPatterns.data]); + + const denoisedRows = useQuery({ + queryKey: [ + 'denoised-rows', + config, + denoiseResults, + // Only include processed rows if denoising is enabled + // This helps prevent the queryKey from getting extremely large + // and causing memory issues, when it's not used. + ...(denoiseResults ? [processedRows] : []), + noisyPatternIds, + patternColumn, + ], + queryFn: async () => { + if (!denoiseResults) { + return []; + } + // No noisy patterns, so no need to denoise + if (noisyPatternIds.length === 0) { + return processedRows; + } + + const matchedLogs = await groupedPatterns.miner?.matchLogs( + processedRows.map(row => row[patternColumn ?? '']), + ); + return processedRows.filter((row, i) => { + const match = matchedLogs?.[i]; + return !noisyPatternIds.includes(`${match}`); + }); + }, + placeholderData: (previousData, previousQuery) => { + // If it's the same search, but new data, return the previous data while we load + if ( + previousQuery?.queryKey?.[0] === 'denoised-rows' && + previousQuery?.queryKey?.[1] === config + ) { + return previousData; + } + return undefined; + }, + gcTime: isLive ? ms('30s') : ms('5m'), // more aggressive gc for live data, since it can end up holding lots of data + enabled: + denoiseResults && + noisyPatterns.isSuccess && + processedRows.length > 0 && + groupedPatterns.miner != null, + }); + + return { + rows: denoiseResults ? (denoisedRows.data ?? []) : processedRows, + noisyPatterns: noisyPatterns.data, + hasNoisyPatterns: noisyPatternIds.length > 0, + isFetching: + denoisedRows.isFetching || + noisyPatterns.isFetching || + groupedPatterns.isLoading, + }; +} + +/** The "Removed Noisy Event Patterns" summary shown above denoised results. */ +function DenoisedPatternsSummary({ + noisyPatterns, + hasNoisyPatterns, +}: { + noisyPatterns: { id: string; pattern: string }[] | undefined; + hasNoisyPatterns: boolean; +}) { + return ( + + + Removed Noisy Event Patterns + + + {noisyPatterns?.map(p => ( + + {p.pattern} + + ))} + {!hasNoisyPatterns && No noisy patterns found} + + + ); +} + function DBSqlRowTableComponent({ config, sourceId, @@ -1736,88 +1910,17 @@ function DBSqlRowTableComponent({ const { data: source } = useSource({ id: sourceId }); const patternColumn = columns[columns.length - 1]; - const groupedPatterns = useGroupedPatterns({ + const denoise = useDenoisedRows({ config, - samples: DENOISE_SAMPLE_SIZE, - bodyValueExpression: patternColumn ?? '', - severityTextExpression: - (source?.kind === SourceKind.Log - ? source.severityTextExpression - : undefined) ?? '', - totalCount: undefined, - enabled: denoiseResults, - }); - const noisyPatterns = useQuery({ - queryKey: ['noisy-patterns', config], - queryFn: async () => { - return Object.values(groupedPatterns.data).filter( - p => - p.count / (groupedPatterns.sampledRowCount ?? 1) > - DENOISE_NOISE_THRESHOLD, - ); - }, - enabled: - denoiseResults && - groupedPatterns.data != null && - Object.values(groupedPatterns.data).length > 0 && - groupedPatterns.miner != null, - }); - const noisyPatternIds = useMemo(() => { - return noisyPatterns.data?.map(p => p.id) ?? []; - }, [noisyPatterns.data]); - - const denoisedRows = useQuery({ - queryKey: [ - 'denoised-rows', - config, - denoiseResults, - // Only include processed rows if denoising is enabled - // This helps prevent the queryKey from getting extremely large - // and causing memory issues, when it's not used. - ...(denoiseResults ? [processedRows] : []), - noisyPatternIds, - patternColumn, - ], - queryFn: async () => { - if (!denoiseResults) { - return []; - } - // No noisy patterns, so no need to denoise - if (noisyPatternIds.length === 0) { - return processedRows; - } - - const matchedLogs = await groupedPatterns.miner?.matchLogs( - processedRows.map(row => row[patternColumn]), - ); - return processedRows.filter((row, i) => { - const match = matchedLogs?.[i]; - return !noisyPatternIds.includes(`${match}`); - }); - }, - placeholderData: (previousData, previousQuery) => { - // If it's the same search, but new data, return the previous data while we load - if ( - previousQuery?.queryKey?.[0] === 'denoised-rows' && - previousQuery?.queryKey?.[1] === config - ) { - return previousData; - } - return undefined; - }, - gcTime: isLive ? ms('30s') : ms('5m'), // more aggressive gc for live data, since it can end up holding lots of data - enabled: - denoiseResults && - noisyPatterns.isSuccess && - processedRows.length > 0 && - groupedPatterns.miner != null, + sourceId, + processedRows, + patternColumn, + denoiseResults, + isLive, }); const isLoading = denoiseResults - ? isFetching || - denoisedRows.isFetching || - noisyPatterns.isFetching || - groupedPatterns.isLoading + ? isFetching || denoise.isFetching : isFetching; const loadingDate = @@ -1828,28 +1931,17 @@ function DBSqlRowTableComponent({ return ( <> {denoiseResults && ( - - - Removed Noisy Event Patterns - - - {noisyPatterns.data?.map(p => ( - - {p.pattern} - - ))} - {noisyPatternIds.length === 0 && ( - No noisy patterns found - )} - - + )} + + {name} + + ); +} diff --git a/packages/app/src/hooks/useOffsetPaginatedQuery.tsx b/packages/app/src/hooks/useOffsetPaginatedQuery.tsx index 5a27d39b96..2dabb32720 100644 --- a/packages/app/src/hooks/useOffsetPaginatedQuery.tsx +++ b/packages/app/src/hooks/useOffsetPaginatedQuery.tsx @@ -427,7 +427,9 @@ function flattenPages(pages: TQueryFnData[]) { return pages.flatMap(p => p.data); } -function flattenData(data: TData | undefined): TQueryFnData | null { +function flattenData( + data: TData | undefined, +): (TQueryFnData & { lastPageRowCount: number }) | null { if (data == null || data.pages.length === 0) { return null; } @@ -437,6 +439,10 @@ function flattenData(data: TData | undefined): TQueryFnData | null { data: flattenPages(data.pages), chSql: data.pages[0].chSql, window: data.pages[data.pages.length - 1].window, + // Whether the last fetched page hit results distinguishes "still mid-window + // at LIMIT" from "window drained" — multi-source merge uses this to compute + // how far this stream's time coverage safely extends. + lastPageRowCount: data.pages[data.pages.length - 1].data.length, }; }