diff --git a/.changeset/source-row-key-expression.md b/.changeset/source-row-key-expression.md new file mode 100644 index 000000000..66c39403d --- /dev/null +++ b/.changeset/source-row-key-expression.md @@ -0,0 +1,9 @@ +--- +'@hyperdx/common-utils': minor +'@hyperdx/api': minor +'@hyperdx/app': minor +--- + +feat: add an optional "Row key expression" to log and trace sources. When set, +the side panel's row lookup selects and filters on only those columns instead of +detecting keys from table metadata. diff --git a/packages/api/openapi.json b/packages/api/openapi.json index a2c7b3fae..8e883b982 100644 --- a/packages/api/openapi.json +++ b/packages/api/openapi.json @@ -5192,6 +5192,12 @@ "nullable": true, "example": "Timestamp, Body, ServiceName" }, + "rowKeyExpression": { + "type": "string", + "description": "Comma-separated expressions that together identify one row, e.g. the partition key, primary key, _block_number and _block_offset. When set, row details are looked up by these columns only. Leave blank to detect keys from table metadata.", + "nullable": true, + "example": "toDate(Timestamp), ServiceName, Timestamp, _block_number, _block_offset" + }, "useTextIndexForImplicitColumn": { "type": "string", "enum": [ @@ -5446,6 +5452,12 @@ "nullable": true, "example": "Timestamp, Body, ServiceName" }, + "rowKeyExpression": { + "type": "string", + "description": "Comma-separated expressions that together identify one row, e.g. the partition key, primary key, _block_number and _block_offset. When set, row details are looked up by these columns only. Leave blank to detect keys from table metadata.", + "nullable": true, + "example": "toDate(Timestamp), SpanName, Timestamp, _block_number, _block_offset" + }, "useTextIndexForImplicitColumn": { "type": "string", "enum": [ diff --git a/packages/api/src/mcp/__tests__/sources.int.test.ts b/packages/api/src/mcp/__tests__/sources.int.test.ts index 59367443b..3f710f169 100644 --- a/packages/api/src/mcp/__tests__/sources.int.test.ts +++ b/packages/api/src/mcp/__tests__/sources.int.test.ts @@ -1451,6 +1451,8 @@ describe('MCP Source Tools', () => { // unwritable through save_source). useTextIndexForImplicitColumn: 'enabled', knownColumnsListExpression: 'TraceId, SpanId', + rowKeyExpression: + 'ServiceName, Timestamp, _block_number, _block_offset', sampleRateExpression: 'SampleRate', highlightedTraceAttributeExpressions: [ { sqlExpression: "SpanAttributes['http.method']", alias: 'method' }, @@ -1480,6 +1482,9 @@ describe('MCP Source Tools', () => { // Advanced fields surface too. expect(config.useTextIndexForImplicitColumn).toBe('enabled'); expect(config.knownColumnsListExpression).toBe('TraceId, SpanId'); + expect(config.rowKeyExpression).toBe( + 'ServiceName, Timestamp, _block_number, _block_offset', + ); expect(config.sampleRateExpression).toBe('SampleRate'); expect(config.highlightedTraceAttributeExpressions).toEqual([ { sqlExpression: "SpanAttributes['http.method']", alias: 'method' }, @@ -1514,6 +1519,9 @@ describe('MCP Source Tools', () => { ); expect(stored?.get('useTextIndexForImplicitColumn')).toBe('enabled'); expect(stored?.get('sampleRateExpression')).toBe('SampleRate'); + expect(stored?.get('rowKeyExpression')).toBe( + 'ServiceName, Timestamp, _block_number, _block_offset', + ); // 4. describe the clone: advanced config round-trips identically. const describedClone = await callTool( diff --git a/packages/api/src/mcp/tools/sources/schemas.ts b/packages/api/src/mcp/tools/sources/schemas.ts index 03f293efa..c095cb6a1 100644 --- a/packages/api/src/mcp/tools/sources/schemas.ts +++ b/packages/api/src/mcp/tools/sources/schemas.ts @@ -254,6 +254,15 @@ const mcpSaveSourceSchema = z.object({ 'targets, used instead of SELECT * when fetching full rows. Column ' + 'names only, no expressions or aliases. Omit to select all columns.', ), + rowKeyExpression: z + .string() + .optional() + .describe( + 'Log and trace sources. Comma-separated expressions that together ' + + 'identify one row, e.g. the partition key, primary key, _block_number ' + + 'and _block_offset. When set, the row side panel looks rows up by ' + + 'these only. Omit to detect keys from table metadata.', + ), orderByExpression: z .string() .optional() diff --git a/packages/api/src/models/source.ts b/packages/api/src/models/source.ts index 6b162640d..42d8e44b7 100644 --- a/packages/api/src/models/source.ts +++ b/packages/api/src/models/source.ts @@ -170,6 +170,7 @@ export const LogSource = Source.discriminator( spanIdExpression: String, implicitColumnExpression: String, knownColumnsListExpression: String, + rowKeyExpression: String, useTextIndexForImplicitColumn: { type: String, enum: Object.values(UseTextIndex), @@ -222,6 +223,7 @@ export const TraceSource = Source.discriminator( spanLinksValueExpression: String, implicitColumnExpression: String, knownColumnsListExpression: String, + rowKeyExpression: String, useTextIndexForImplicitColumn: { type: String, enum: Object.values(UseTextIndex), diff --git a/packages/api/src/routers/external-api/v2/sources.ts b/packages/api/src/routers/external-api/v2/sources.ts index 561625bc6..8da4fd0b4 100644 --- a/packages/api/src/routers/external-api/v2/sources.ts +++ b/packages/api/src/routers/external-api/v2/sources.ts @@ -406,6 +406,11 @@ function formatExternalSource(source: SourceDocument) { * description: For Distributed table sources whose target tables have non-matching column sets. A list of columns supported across all target tables, used instead of SELECT * when fetching full row data. Leave blank to select all columns. * nullable: true * example: Timestamp, Body, ServiceName + * rowKeyExpression: + * type: string + * description: Comma-separated expressions that together identify one row, e.g. the partition key, primary key, _block_number and _block_offset. When set, row details are looked up by these columns only. Leave blank to detect keys from table metadata. + * nullable: true + * example: toDate(Timestamp), ServiceName, Timestamp, _block_number, _block_offset * useTextIndexForImplicitColumn: * type: string * enum: [auto, enabled, disabled] @@ -607,6 +612,11 @@ function formatExternalSource(source: SourceDocument) { * description: For Distributed table sources whose target tables have non-matching column sets. A list of columns supported across all target tables, used instead of SELECT * when fetching full row data. Leave blank to select all columns. * nullable: true * example: Timestamp, Body, ServiceName + * rowKeyExpression: + * type: string + * description: Comma-separated expressions that together identify one row, e.g. the partition key, primary key, _block_number and _block_offset. When set, row details are looked up by these columns only. Leave blank to detect keys from table metadata. + * nullable: true + * example: toDate(Timestamp), SpanName, Timestamp, _block_number, _block_offset * useTextIndexForImplicitColumn: * type: string * enum: [auto, enabled, disabled] diff --git a/packages/app/src/components/DBRowTable.tsx b/packages/app/src/components/DBRowTable.tsx index 2c90cbf20..c20b1586e 100644 --- a/packages/app/src/components/DBRowTable.tsx +++ b/packages/app/src/components/DBRowTable.tsx @@ -31,6 +31,8 @@ import { import { splitAndTrimWithBracket } from '@hyperdx/common-utils/dist/core/utils'; import { BuilderChartConfigWithDateRange, + isLogSource, + isTraceSource, SelectList, TSource, } from '@hyperdx/common-utils/dist/types'; @@ -1576,15 +1578,27 @@ function getSelectLength(select: SelectList): number { } } +function getRowKeyExpression(source: TSource | undefined): string | undefined { + return source && (isLogSource(source) || isTraceSource(source)) + ? source.rowKeyExpression?.trim() || undefined + : undefined; +} + export function useConfigWithAdditionalSelect( config: BuilderChartConfigWithDateRange, sourceId?: string, + rowKeyExpression?: string, ) { - const { data: tableMetadata } = useTableMetadata({ - databaseName: config.from.databaseName, - tableName: config.from.tableName, - connectionId: config.connection, - }); + const hasRowKeyOverride = !!sourceId && !!rowKeyExpression; + + const { data: tableMetadata } = useTableMetadata( + { + databaseName: config.from.databaseName, + tableName: config.from.tableName, + connectionId: config.connection, + }, + hasRowKeyOverride ? { enabled: false } : undefined, + ); // Only check for row-ID columns for row-level queries (sourceId present). // Skip for aggregate queries (e.g. patterns) where extra keys are irrelevant. @@ -1594,12 +1608,13 @@ export function useConfigWithAdditionalSelect( tableName: config.from.tableName, connectionId: config.connection, }, - { enabled: !!sourceId }, + { enabled: !!sourceId && !hasRowKeyOverride }, ); const engineFull = tableMetadata?.engine_full ?? ''; const needsMergeTreeSettings = !!sourceId && + !hasRowKeyOverride && !!tableMetadata?.engine?.includes('MergeTree') && omitsBlockColumnSetting(engineFull); @@ -1612,6 +1627,19 @@ export function useConfigWithAdditionalSelect( const partitionKey = tableMetadata?.partition_key; return useMemo(() => { + if (sourceId && rowKeyExpression) { + const { select, additionalKeysLength } = appendSelectWithAdditionalKeys( + config.select, + rowKeyExpression, + '', + ); + const rowKeyColumns = new Set([ + ...splitAndTrimWithBracket(rowKeyExpression), + ...extractColumnReferencesFromKey(rowKeyExpression), + ]); + return { ...config, select, additionalKeysLength, rowKeyColumns }; + } + // Wait for settings to avoid querying twice. if ( primaryKey == null || @@ -1674,6 +1702,7 @@ export function useConfigWithAdditionalSelect( columns, mergeTreeSettings, sourceId, + rowKeyExpression, ]); } @@ -1821,7 +1850,12 @@ function DBSqlRowTableComponent({ return base; }, [me, config, orderByArray]); - const mergedConfig = useConfigWithAdditionalSelect(mergedConfigObj, sourceId); + const { data: source } = useSource({ id: sourceId }); + const mergedConfig = useConfigWithAdditionalSelect( + mergedConfigObj, + sourceId, + getRowKeyExpression(source), + ); const { data, fetchNextPage, hasNextPage, isFetching, isError, error } = useOffsetPaginatedQuery(mergedConfig ?? config, { @@ -1943,7 +1977,6 @@ function DBSqlRowTableComponent({ } }, [data?.meta, onResolvedColumnsChange]); - const { data: source } = useSource({ id: sourceId }); const denoised = useDenoisedRows({ config, source, diff --git a/packages/app/src/components/Sources/SourceForm/LogTableModelForm.tsx b/packages/app/src/components/Sources/SourceForm/LogTableModelForm.tsx index b936c352f..58c2826ba 100644 --- a/packages/app/src/components/Sources/SourceForm/LogTableModelForm.tsx +++ b/packages/app/src/components/Sources/SourceForm/LogTableModelForm.tsx @@ -14,6 +14,7 @@ import { DEFAULT_DATABASE, EVENT_NAME_EXPRESSION_HELP_TEXT, KNOWN_COLUMNS_EXPRESSION_HELP_TEXT, + ROW_KEY_EXPRESSION_HELP_TEXT, SERVICE_VERSION_EXPRESSION_HELP_TEXT, SEVERITY_NUMBER_EXPRESSION_HELP_TEXT, } from './constants'; @@ -285,6 +286,22 @@ export function LogTableModelForm(props: TableModelProps) { disableKeywordAutocomplete /> + + + + + + = { export const KNOWN_COLUMNS_EXPRESSION_HELP_TEXT = 'For Distributed table sources whose target tables have non-matching column sets. Provide a list of columns supported across all target tables; it is used instead of SELECT * when fetching full row data (e.g. the row side panel). Leave blank to select all columns. This should be a comma-separated list of column names - do not include non-column expressions or aliases.'; +export const ROW_KEY_EXPRESSION_HELP_TEXT = + 'Comma-separated columns that together identify one row, e.g. the partition key, primary key, _block_number and _block_offset. When set, row details are looked up by these columns only.'; + export const SERVICE_VERSION_EXPRESSION_HELP_TEXT = "Identifies the running release of a service. Defaults to the OpenTelemetry service.version resource attribute. Point it elsewhere if your release identifier lives in another attribute - under GitOps it is often the container image tag. If services in this table use different attributes, fall back across them: coalesce(nullIf(ResourceAttributes['service.version'], ''), nullIf(ResourceAttributes['container.image.tag'], ''))."; diff --git a/packages/app/src/components/__tests__/DBRowTable.test.tsx b/packages/app/src/components/__tests__/DBRowTable.test.tsx index b6232dbbc..8b8290942 100644 --- a/packages/app/src/components/__tests__/DBRowTable.test.tsx +++ b/packages/app/src/components/__tests__/DBRowTable.test.tsx @@ -1198,4 +1198,85 @@ describe('useConfigWithAdditionalSelect', () => { 'Timestamp,ServiceName,Body,toDate(Timestamp),toStartOfFiveMinutes(Timestamp),_block_offset', ); }); + + describe('with a row key expression', () => { + const metadataLoading = () => { + mockUseTableMetadata.mockReturnValue({ data: undefined }); + mockUseColumns.mockReturnValue({ data: undefined }); + mockUseMergeTreeSettings.mockReturnValue({ + data: undefined, + isLoading: true, + }); + }; + + it('restricts the row lookup to the listed columns', () => { + metadataLoading(); + + const { result } = renderHook(() => + useConfigWithAdditionalSelect( + config, + 'source-id', + 'ServiceName, Timestamp, _block_number, _block_offset', + ), + ); + + expect(result.current?.select).toBe( + 'Timestamp,ServiceName,Body,_block_number,_block_offset', + ); + expect(result.current?.additionalKeysLength).toBe(2); + expect(result.current?.rowKeyColumns).toEqual( + new Set(['ServiceName', 'Timestamp', '_block_number', '_block_offset']), + ); + expect(result.current?.rowKeyColumns?.has('Body')).toBe(false); + }); + + it('returns a config without waiting for table metadata or settings', () => { + metadataLoading(); + + const { result } = renderHook(() => + useConfigWithAdditionalSelect( + config, + 'source-id', + 'Timestamp, _block_number, _block_offset', + ), + ); + + expect(mockUseTableMetadata).toHaveBeenCalledWith(expect.anything(), { + enabled: false, + }); + expect(mockUseColumns).toHaveBeenCalledWith(expect.anything(), { + enabled: false, + }); + expect(mockUseMergeTreeSettings).toHaveBeenCalledWith('conn', { + enabled: false, + }); + expect(result.current).toBeDefined(); + }); + + it('does not split inside function calls', () => { + metadataLoading(); + + const { result } = renderHook(() => + useConfigWithAdditionalSelect( + config, + 'source-id', + 'toYYYYMM(event_date), service, _block_number, _block_offset', + ), + ); + + expect(result.current?.select).toBe( + 'Timestamp,ServiceName,Body,event_date,service,_block_number,_block_offset,toYYYYMM(event_date)', + ); + expect(result.current?.additionalKeysLength).toBe(5); + expect(result.current?.rowKeyColumns).toEqual( + new Set([ + 'toYYYYMM(event_date)', + 'service', + '_block_number', + '_block_offset', + 'event_date', + ]), + ); + }); + }); }); diff --git a/packages/common-utils/src/types.ts b/packages/common-utils/src/types.ts index dba9c7854..d21d64df0 100644 --- a/packages/common-utils/src/types.ts +++ b/packages/common-utils/src/types.ts @@ -2737,6 +2737,11 @@ export const LogSourceSchema = BaseSourceSchema.extend({ spanIdExpression: z.string().optional(), implicitColumnExpression: z.string().optional(), knownColumnsListExpression: z.string().optional(), + /** + * Comma-separated expressions that together identify one row. When set, the + * row side panel looks rows up by these instead of table metadata. + */ + rowKeyExpression: z.string().optional(), /** * @deprecated Application-side SQL predicate AND'd into every query against * the source. Not a security boundary; bypassable by direct table SELECT. @@ -2791,6 +2796,8 @@ export const TraceSourceSchema = BaseSourceSchema.extend({ spanLinksValueExpression: z.string().optional(), implicitColumnExpression: z.string().optional(), knownColumnsListExpression: z.string().optional(), + /** See `rowKeyExpression` on `LogSourceSchema`. */ + rowKeyExpression: z.string().optional(), displayedTimestampValueExpression: z.string().optional(), highlightedTraceAttributeExpressions: HighlightedAttributeExpressionsSchema.optional(),