diff --git a/.changeset/adhoc-filters-external-api.md b/.changeset/adhoc-filters-external-api.md new file mode 100644 index 0000000000..9edb98ee56 --- /dev/null +++ b/.changeset/adhoc-filters-external-api.md @@ -0,0 +1,10 @@ +--- +'@hyperdx/api': minor +--- + +feat: Create, read, and update ad hoc dashboard filters through the external API + +`/api/v2/dashboards` now accepts and returns `ADHOC` filters, with SQL and +PromQL source types, and their `adhoc` entries in `savedFilterValues`. Each +filter's sources must exist and be a kind its `sourceType` supports. MCP still +hides ad hoc filters. diff --git a/packages/api/openapi.json b/packages/api/openapi.json index a2c7b3faea..0bd7c40a81 100644 --- a/packages/api/openapi.json +++ b/packages/api/openapi.json @@ -959,13 +959,16 @@ "description": "Query language for the where clause." }, "SavedFilterValue": { - "description": "A single saved dashboard filter selection. Either a rendered SQL condition, or a selection addressed by the name of the dashboard variable it belongs to.\n", + "description": "A single saved dashboard filter selection. Either a rendered SQL condition, a selection addressed by the name of the dashboard variable it belongs to, or the conditions of an ADHOC filter.\n", "oneOf": [ { "$ref": "#/components/schemas/SqlSavedFilterValue" }, { "$ref": "#/components/schemas/VariableSavedFilterValue" + }, + { + "$ref": "#/components/schemas/AdhocSavedFilterValue" } ] }, @@ -1029,6 +1032,78 @@ } } }, + "AdhocSavedFilterValue": { + "type": "object", + "required": [ + "type", + "name", + "conditions" + ], + "properties": { + "type": { + "type": "string", + "enum": [ + "adhoc" + ], + "description": "Filter type.", + "example": "adhoc" + }, + "name": { + "type": "string", + "minLength": 1, + "maxLength": 1024, + "description": "The variableName of the ADHOC filter these conditions belong to.", + "example": "conditions" + }, + "conditions": { + "type": "array", + "maxItems": 100, + "description": "The filter's conditions.", + "items": { + "$ref": "#/components/schemas/AdhocFilterCondition" + } + } + } + }, + "AdhocFilterCondition": { + "type": "object", + "description": "One \"key operator value\" condition. Conditions are AND'd, except that IN (sql) and =| (promql) conditions sharing a key are OR'd into one.\n", + "required": [ + "key", + "operator", + "value" + ], + "properties": { + "key": { + "type": "string", + "minLength": 1, + "maxLength": 10000, + "description": "A SQL expression (column or map access) for sql filters, or a label name for promql filters.", + "example": "ServiceName" + }, + "operator": { + "type": "string", + "enum": [ + "=", + "!=", + "=~", + "!~", + "LIKE", + "NOT LIKE", + "IN", + "=|" + ], + "description": "Comparison operator. =~ and !~ match a regular expression. LIKE, NOT LIKE, and IN apply to sql filters only, and =| to promql filters only; a condition its filter's sourceType can't express is not applied.\n", + "example": "=" + }, + "value": { + "type": "string", + "maxLength": 10000, + "description": "Value, pattern, or regular expression to compare against.", + "example": "api" + } + } + }, "MetricDataType": { "type": "string", "enum": [ @@ -3951,6 +4026,9 @@ }, { "$ref": "#/components/schemas/PrometheusLabelFilterInput" + }, + { + "$ref": "#/components/schemas/AdhocFilterInput" } ], "discriminator": { @@ -3958,7 +4036,8 @@ "mapping": { "QUERY_EXPRESSION": "#/components/schemas/QueryExpressionFilterInput", "STATIC_LIST": "#/components/schemas/StaticListFilterInput", - "PROMETHEUS_LABEL": "#/components/schemas/PrometheusLabelFilterInput" + "PROMETHEUS_LABEL": "#/components/schemas/PrometheusLabelFilterInput", + "ADHOC": "#/components/schemas/AdhocFilterInput" } } }, @@ -4262,6 +4341,101 @@ } } }, + "AdhocFilterInput": { + "type": "object", + "description": "A filter holding free-form \"key operator value\" conditions, with keys\nand values suggested from its sources: columns and map keys for sql\nsources, labels for promql sources. Its conditions are always\nbroadcast to the tiles it applies to: as WHERE conditions on builder\ntiles and raw sql tiles using the $__filters macro, or as label\nmatchers on PromQL tiles. Its conditions are saved in\nsavedFilterValues as an \"adhoc\" entry. Ad hoc filters are a preview\nfeature.\n", + "required": [ + "type", + "name", + "sourceType", + "sources" + ], + "properties": { + "type": { + "type": "string", + "enum": [ + "ADHOC" + ], + "description": "Discriminator. Must be \"ADHOC\".", + "example": "ADHOC" + }, + "name": { + "type": "string", + "minLength": 1, + "description": "Display name for the dashboard filter", + "example": "Conditions" + }, + "sourceType": { + "type": "string", + "enum": [ + "sql", + "promql" + ], + "description": "The kind of sources the filter reads keys from and applies to: Log,\nTrace, or Session sources for sql, PromQL sources for promql.\n", + "example": "sql" + }, + "sources": { + "type": "array", + "minItems": 1, + "items": { + "type": "string" + }, + "description": "IDs of the sources whose keys and values the filter suggests, all of sourceType.", + "example": [ + "65f5e4a3b9e77c001a111111" + ] + }, + "where": { + "type": "string", + "description": "Optional condition narrowing the rows keys and values are suggested\nfrom. Never applied to tiles. May reference the dashboard's\nvariables. sql filters only: rejected when sourceType is promql.\n", + "example": "ServiceName:api" + }, + "whereLanguage": { + "type": "string", + "enum": [ + "sql", + "lucene" + ], + "description": "Language of the where condition. Rejected when sourceType is promql.", + "example": "lucene" + }, + "appliesToSourceIds": { + "type": "array", + "items": { + "type": "string" + }, + "description": "IDs of the sources whose tiles the conditions apply to, all of\nsourceType. Omit or send an empty array to apply to the tiles of\nthe filter's sources.\n", + "example": [ + "65f5e4a3b9e77c001a111111" + ] + }, + "isBroadcastEnabled": { + "type": "boolean", + "enum": [ + true + ], + "default": true, + "description": "Must be true or omitted.", + "example": true + }, + "isVariableEnabled": { + "type": "boolean", + "enum": [ + true + ], + "default": true, + "description": "Must be true or omitted.", + "example": true + }, + "variableName": { + "type": "string", + "maxLength": 64, + "pattern": "^[a-zA-Z][a-zA-Z0-9_]*$", + "description": "Unique name the filter's conditions are saved under, in\nsavedFilterValues and the dashboard URL. Tiles can't reference it\nas a variable. Defaults to the display name with whitespace\nreplaced by underscores and remaining illegal characters removed.\nMust be unique across a dashboard's variable-enabled filters.\n", + "example": "conditions" + } + } + }, "Filter": { "allOf": [ { diff --git a/packages/api/src/mcp/tools/dashboards/getDashboard.ts b/packages/api/src/mcp/tools/dashboards/getDashboard.ts index a18f09952b..2133ec09b3 100644 --- a/packages/api/src/mcp/tools/dashboards/getDashboard.ts +++ b/packages/api/src/mcp/tools/dashboards/getDashboard.ts @@ -63,6 +63,7 @@ export function registerGetDashboard({ } const externalDashboard = convertToExternalDashboard(dashboard, { hidePromql: !isPromqlEnabled(context), + hideAdhoc: true, }); return { content: [ diff --git a/packages/api/src/mcp/tools/dashboards/getDashboardTile.ts b/packages/api/src/mcp/tools/dashboards/getDashboardTile.ts index f9a4d2632f..7d331d965c 100644 --- a/packages/api/src/mcp/tools/dashboards/getDashboardTile.ts +++ b/packages/api/src/mcp/tools/dashboards/getDashboardTile.ts @@ -51,6 +51,7 @@ export function registerGetDashboardTile({ const externalDashboard = convertToExternalDashboard(dashboard, { hidePromql: !isPromqlEnabled(context), + hideAdhoc: true, }); const tile = externalDashboard.tiles.find(t => t.id === tileId); if (!tile) { diff --git a/packages/api/src/mcp/tools/dashboards/patchDashboard.ts b/packages/api/src/mcp/tools/dashboards/patchDashboard.ts index aeea54d7bd..4488ca1c7b 100644 --- a/packages/api/src/mcp/tools/dashboards/patchDashboard.ts +++ b/packages/api/src/mcp/tools/dashboards/patchDashboard.ts @@ -15,7 +15,7 @@ import { isConfigTile, validateDashboardTiles, } from '@/routers/external-api/v2/utils/dashboards'; -import { getExternalCompatibleFilterValues } from '@/utils/externalApi'; +import { getMcpCompatibleFilterValues } from '@/utils/externalApi'; import type { ExternalDashboardTileWithId } from '@/utils/zod'; import { externalDashboardTileSchemaWithId } from '@/utils/zod'; @@ -127,7 +127,7 @@ export function registerPatchDashboard({ // Convert only for the error message (read-only, no write-back). const externalDashboard = convertToExternalDashboard( existingDashboard, - { hidePromql: !promqlEnabled }, + { hidePromql: !promqlEnabled, hideAdhoc: true }, ); return mcpUserError( `Tile not found: ${tileId}. Available tile IDs: ${externalDashboard.tiles.map(t => `${t.id} (${t.name})`).join(', ')}`, @@ -271,7 +271,7 @@ export function registerPatchDashboard({ tags: updatedDashboard.tags, ...(savedFilterValues !== undefined ? { - savedFilterValues: getExternalCompatibleFilterValues( + savedFilterValues: getMcpCompatibleFilterValues( updatedDashboard.savedFilterValues, ), } diff --git a/packages/api/src/mcp/tools/dashboards/queryTile.ts b/packages/api/src/mcp/tools/dashboards/queryTile.ts index b6cd385b57..0f9df6b116 100644 --- a/packages/api/src/mcp/tools/dashboards/queryTile.ts +++ b/packages/api/src/mcp/tools/dashboards/queryTile.ts @@ -78,6 +78,7 @@ export function registerQueryTile({ const externalDashboard = convertToExternalDashboard(dashboard, { hidePromql: !isPromqlEnabled(context), + hideAdhoc: true, }); const tile = externalDashboard.tiles.find(t => t.id === tileId); if (!tile) { diff --git a/packages/api/src/mcp/tools/dashboards/queryTiles.ts b/packages/api/src/mcp/tools/dashboards/queryTiles.ts index 44af35c950..9860ad768c 100644 --- a/packages/api/src/mcp/tools/dashboards/queryTiles.ts +++ b/packages/api/src/mcp/tools/dashboards/queryTiles.ts @@ -184,6 +184,7 @@ export function registerQueryTiles({ const externalDashboard = convertToExternalDashboard(dashboard, { hidePromql: !isPromqlEnabled(context), + hideAdhoc: true, }); const allTiles = externalDashboard.tiles; diff --git a/packages/api/src/mcp/tools/dashboards/saveDashboard.ts b/packages/api/src/mcp/tools/dashboards/saveDashboard.ts index ba0c0d21e3..bde01373e8 100644 --- a/packages/api/src/mcp/tools/dashboards/saveDashboard.ts +++ b/packages/api/src/mcp/tools/dashboards/saveDashboard.ts @@ -236,6 +236,7 @@ async function createDashboard({ const externalDashboard = convertToExternalDashboard(newDashboard, { hidePromql: !promqlEnabled, + hideAdhoc: true, }); return { content: [ @@ -452,6 +453,7 @@ async function updateDashboard({ const externalDashboard = convertToExternalDashboard(updatedDashboard, { hidePromql: !promqlEnabled, + hideAdhoc: true, }); return { content: [ diff --git a/packages/api/src/mcp/tools/dashboards/validation.ts b/packages/api/src/mcp/tools/dashboards/validation.ts index 987ab6ad2f..58d82da8de 100644 --- a/packages/api/src/mcp/tools/dashboards/validation.ts +++ b/packages/api/src/mcp/tools/dashboards/validation.ts @@ -18,6 +18,7 @@ import { TIME_RANGE_MACROS, } from '@hyperdx/common-utils/dist/macros'; import { + DashboardFilterValue, DisplayType, QueryExpressionDashboardFilter, SavedChartConfig, @@ -36,8 +37,6 @@ import { } from '@/routers/external-api/v2/utils/dashboards'; import type { ExternalDashboardTileWithId } from '@/utils/zod'; -import type { McpSavedFilterValues } from './schemas'; - /** * Returns one entry per raw SQL tile that uses a source-dependent macro but * does not set a `sourceId`, recording which macro(s) triggered it so callers @@ -253,7 +252,7 @@ export function getFilterVariableWarnings( * applies. Warn so the agent can fix the name or add the filter. */ export function getSavedFilterValueWarnings( - savedFilterValues: McpSavedFilterValues | undefined, + savedFilterValues: DashboardFilterValue[] | undefined, filters: FilterForVariableDeclaration[] | undefined, ): string[] { if (!savedFilterValues?.length) return []; diff --git a/packages/api/src/routers/api/dashboards.ts b/packages/api/src/routers/api/dashboards.ts index 544679ef92..d7bf0a82f8 100644 --- a/packages/api/src/routers/api/dashboards.ts +++ b/packages/api/src/routers/api/dashboards.ts @@ -3,11 +3,6 @@ import { validateDashboardFilterOptionUniqueness, validateDashboardFilterVariableNames, } from '@hyperdx/common-utils/dist/dashboardValidation'; -import { - getAdhocFilterSourceKinds, - isAdhocFilter, - isPrometheusLabelFilter, -} from '@hyperdx/common-utils/dist/filters'; import { DashboardFilter, DashboardSchema, @@ -15,7 +10,6 @@ import { PresetDashboard, PresetDashboardFilterSchema, resolveChartPaletteToken, - SourceKind, walkRawDashboardTileColors, } from '@hyperdx/common-utils/dist/types'; import express from 'express'; @@ -39,6 +33,10 @@ import { import { getSources } from '@/controllers/sources'; import { getNonNullUserWithTeam } from '@/middleware/auth'; import type { ObjectId } from '@/models'; +import { + getChangedFilterSourceRefs, + getFilterSourceRefError, +} from '@/utils/filterSources'; import { objectIdSchema } from '@/utils/zod'; // create routes that will get and update dashboards @@ -64,22 +62,6 @@ const addFilterIssues = ( validateDashboardFilterOptionUniqueness(data.filters ?? [], ctx); }; -/** The sources a filter reads from, each with the kinds it must be. */ -function getFilterSourceRefs( - filter: DashboardFilter, -): { id: string; kinds: SourceKind[] }[] { - if (isPrometheusLabelFilter(filter)) { - return [{ id: filter.source, kinds: [SourceKind.Promql] }]; - } - if (isAdhocFilter(filter)) { - const kinds = getAdhocFilterSourceKinds(filter.sourceType); - return [...filter.sources, ...(filter.appliesToSourceIds ?? [])].map( - id => ({ id, kinds }), - ); - } - return []; -} - /** * Rejects PROMETHEUS_LABEL and ADHOC filters that reference a source that is * missing or of the wrong kind. Resolving these filters needs a live source of @@ -93,33 +75,9 @@ async function validateFilterSources( filters: DashboardFilter[] = [], existingFilters: DashboardFilter[] = [], ): Promise { - const refKey = ({ id, kinds }: { id: string; kinds: SourceKind[] }) => - `${id}:${kinds.join(',')}`; - const existingRefsById = new Map( - existingFilters.map(filter => [ - filter.id, - new Set(getFilterSourceRefs(filter).map(refKey)), - ]), - ); - const newRefs = filters.flatMap(filter => - getFilterSourceRefs(filter).filter( - ref => !existingRefsById.get(filter.id)?.has(refKey(ref)), - ), - ); - if (newRefs.length === 0) return null; - - const sources = await getSources(teamId.toString()); - const kindById = new Map(sources.map(s => [s._id.toString(), s.kind])); - const invalid = new Set( - newRefs - .filter(({ id, kinds }) => { - const kind = kindById.get(id); - return kind === undefined || !kinds.includes(kind); - }) - .map(({ id }) => id), - ); - if (invalid.size === 0) return null; - return `The following filter source IDs do not exist or are not a kind of source the filter supports: ${[...invalid].join(', ')}`; + const refs = getChangedFilterSourceRefs(filters, existingFilters); + if (refs.length === 0) return null; + return getFilterSourceRefError(refs, await getSources(teamId.toString())); } /** diff --git a/packages/api/src/routers/external-api/__tests__/dashboards.int.test.ts b/packages/api/src/routers/external-api/__tests__/dashboards.int.test.ts index 93fea0e4d5..2149bedc08 100644 --- a/packages/api/src/routers/external-api/__tests__/dashboards.int.test.ts +++ b/packages/api/src/routers/external-api/__tests__/dashboards.int.test.ts @@ -6492,7 +6492,7 @@ describe('External API v2 Dashboards - new format', () => { expect(response.status).toBe(400); expect(JSON.stringify(response.body)).toContain( - `PROMETHEUS_LABEL filters require a PromQL source. The following source IDs are not PromQL sources: ${traceSource._id.toString()}`, + `The following filter source IDs do not exist or are not a kind of source the filter supports: ${traceSource._id.toString()}`, ); }); @@ -6502,10 +6502,28 @@ describe('External API v2 Dashboards - new format', () => { expect(response.status).toBe(400); expect(JSON.stringify(response.body)).toContain( - `Could not find the following source IDs: ${sourceId}`, + `The following filter source IDs do not exist or are not a kind of source the filter supports: ${sourceId}`, ); }); + it('accepts a GET response PUT back after its source was deleted', async () => { + const dashboard = await createTestDashboard({ + filters: [ + { + ...omit(promqlFilterInput(), 'sourceId'), + source: new ObjectId().toString(), + }, + ], + }); + const { data } = ( + await authRequest('get', `${BASE_URL}/${dashboard._id}`).expect(200) + ).body; + + await authRequest('put', `${BASE_URL}/${dashboard._id}`) + .send({ ...omit(data, 'id'), name: 'Renamed' }) + .expect(200); + }); + it.each([ ['broadcast enabled', { isBroadcastEnabled: true }], ['not variable-enabled', { isVariableEnabled: false }], @@ -8456,9 +8474,16 @@ describe('External API v2 Dashboards - new format', () => { conditions: [{ key: 'ServiceName', operator: '=', value: 'api' }], }; - it('hides ad hoc filters and their saved values from GET', async () => { + const promqlAdhocFilter = () => ({ + ...adhocFilter(), + sourceType: 'promql' as const, + sources: [promqlSource._id.toString()], + }); + + it('returns ad hoc filters and their saved values from GET', async () => { + const filter = adhocFilter(); const dashboard = await createTestDashboard({ - filters: [adhocFilter()], + filters: [filter], savedFilterValues: [adhocValue], }); @@ -8467,25 +8492,249 @@ describe('External API v2 Dashboards - new format', () => { `${BASE_URL}/${dashboard._id}`, ).expect(200); - expect(response.body.data.filters).toEqual([]); - expect(response.body.data.savedFilterValues).toEqual([]); + expect(response.body.data.filters).toEqual([filter]); + expect(response.body.data.savedFilterValues).toEqual([adhocValue]); + }); + + it('creates SQL and PromQL ad hoc filters with saved conditions', async () => { + const promqlValue = { + type: 'adhoc' as const, + name: 'prom', + conditions: [{ key: 'job', operator: '=|', value: 'api' }], + }; + const response = await authRequest('post', BASE_URL) + .send( + createMockDashboard(traceSource._id.toString(), { + filters: [ + omit( + { ...adhocFilter(), appliesToSourceIds: [traceSource._id] }, + 'id', + ), + // The mode flags default when omitted. + omit( + { ...promqlAdhocFilter(), name: 'Prom', variableName: 'prom' }, + 'id', + 'isBroadcastEnabled', + 'isVariableEnabled', + ), + ], + savedFilterValues: [adhocValue, promqlValue], + }), + ) + .expect(200); + + expect(response.body.data.filters).toEqual([ + { + ...adhocFilter(), + id: expect.any(String), + appliesToSourceIds: [traceSource._id.toString()], + }, + { + ...promqlAdhocFilter(), + id: expect.any(String), + name: 'Prom', + variableName: 'prom', + }, + ]); + expect(response.body.data.savedFilterValues).toEqual([ + adhocValue, + promqlValue, + ]); + + const stored = await Dashboard.findById(response.body.data.id).lean(); + expect(stored?.filters?.[1]).toMatchObject({ + type: 'ADHOC', + sourceType: 'promql', + sources: [promqlSource._id.toString()], + isBroadcastEnabled: true, + }); + }); + + it('round-trips a GET response through PUT', async () => { + const dashboard = await createTestDashboard({ + filters: [adhocFilter()], + savedFilterValues: [adhocValue], + }); + const { data } = ( + await authRequest('get', `${BASE_URL}/${dashboard._id}`).expect(200) + ).body; + + const response = await authRequest('put', `${BASE_URL}/${dashboard._id}`) + .send(omit(data, 'id')) + .expect(200); + + expect(response.body.data.filters).toEqual(data.filters); + expect(response.body.data.savedFilterValues).toEqual( + data.savedFilterValues, + ); + }); + + describe('with a source deleted since the filter was saved', () => { + const deleted = new ObjectId().toString(); + const createWithDeletedSource = () => + createTestDashboard({ + filters: [ + { + ...adhocFilter(), + sources: [traceSource._id.toString(), deleted], + }, + ], + }); + + it('accepts a GET response PUT back unchanged', async () => { + const dashboard = await createWithDeletedSource(); + const { data } = ( + await authRequest('get', `${BASE_URL}/${dashboard._id}`).expect(200) + ).body; + + await authRequest('put', `${BASE_URL}/${dashboard._id}`) + .send({ ...omit(data, 'id'), name: 'Renamed' }) + .expect(200); + }); + + it('rejects the source on a filter that did not reference it', async () => { + const dashboard = await createWithDeletedSource(); + const { data } = ( + await authRequest('get', `${BASE_URL}/${dashboard._id}`).expect(200) + ).body; + + const response = await authRequest( + 'put', + `${BASE_URL}/${dashboard._id}`, + ) + .send({ + ...omit(data, 'id'), + filters: [ + ...data.filters, + // A new filter, with a fresh ID, referencing the deleted source. + { + ...adhocFilter(), + name: 'New', + variableName: 'new', + appliesToSourceIds: [deleted], + }, + ], + }) + .expect(400); + + expect(response.body.message).toContain(deleted); + }); }); - it('rejects an ad hoc filter in a POST', async () => { + it.each([ + ['broadcast disabled', { isBroadcastEnabled: false }], + ['variables disabled', { isVariableEnabled: false }], + ['no sources', { sources: [] }], + ['an invalid source ID', { sources: ['not-an-id'] }], + ['an unknown source type', { sourceType: 'lucene' }], + ['a selection limit', { maxSelections: 1 }], + ['the external sourceId field', { sourceId: new ObjectId().toString() }], + ])('rejects an ad hoc filter with %s', async (_label, overrides) => { await authRequest('post', BASE_URL) .send( createMockDashboard(traceSource._id.toString(), { - filters: [omit(adhocFilter(), 'id')], + filters: [omit({ ...adhocFilter(), ...overrides }, 'id')], + }), + ) + .expect(400); + }); + + it.each([ + ['where', { where: 'job:api' }], + ['whereLanguage', { whereLanguage: 'sql' }], + ])('rejects %s on a PromQL ad hoc filter', async (field, overrides) => { + const response = await authRequest('post', BASE_URL) + .send( + createMockDashboard(traceSource._id.toString(), { + filters: [omit({ ...promqlAdhocFilter(), ...overrides }, 'id')], }), ) .expect(400); + + expect(response.body.message).toContain(`sets ${field}`); }); - it('rejects ad hoc saved values in a POST', async () => { + it('drops a stored where from a PromQL ad hoc filter, so GET -> PUT round-trips', async () => { + const dashboard = await createTestDashboard({ + filters: [ + { ...promqlAdhocFilter(), where: 'job:api', whereLanguage: 'lucene' }, + ], + }); + const { data } = ( + await authRequest('get', `${BASE_URL}/${dashboard._id}`).expect(200) + ).body; + expect(data.filters[0]).not.toHaveProperty('where'); + expect(data.filters[0]).not.toHaveProperty('whereLanguage'); + + await authRequest('put', `${BASE_URL}/${dashboard._id}`) + .send(omit(data, 'id')) + .expect(200); + }); + + it('rejects an ad hoc filter over a missing source', async () => { + const missing = new ObjectId().toString(); + const response = await authRequest('post', BASE_URL) + .send( + createMockDashboard(traceSource._id.toString(), { + filters: [omit({ ...adhocFilter(), sources: [missing] }, 'id')], + }), + ) + .expect(400); + + expect(response.body.message).toContain(missing); + }); + + it.each([ + ['a SQL filter over a PromQL source', () => adhocFilter(), 'promql'], + ['a SQL filter over a Metric source', () => adhocFilter(), 'metric'], + ['a PromQL filter over a Trace source', promqlAdhocFilter, 'trace'], + ] as const)('rejects %s', async (_label, makeFilter, kind) => { + const sourceId = { + promql: promqlSource, + metric: metricSource, + trace: traceSource, + }[kind]._id.toString(); + const response = await authRequest('post', BASE_URL) + .send( + createMockDashboard(traceSource._id.toString(), { + filters: [omit({ ...makeFilter(), sources: [sourceId] }, 'id')], + }), + ) + .expect(400); + + expect(response.body.message).toContain(sourceId); + }); + + it('rejects an ad hoc filter applying to a source of the wrong kind', async () => { + const response = await authRequest('post', BASE_URL) + .send( + createMockDashboard(traceSource._id.toString(), { + filters: [ + omit( + { + ...adhocFilter(), + appliesToSourceIds: [promqlSource._id.toString()], + }, + 'id', + ), + ], + }), + ) + .expect(400); + + expect(response.body.message).toContain(promqlSource._id.toString()); + }); + + it('rejects ad hoc saved values with an unknown operator', async () => { await authRequest('post', BASE_URL) .send( createMockDashboard(traceSource._id.toString(), { - savedFilterValues: [adhocValue], + savedFilterValues: [ + { + ...adhocValue, + conditions: [{ key: 'a', operator: '>', value: '1' }], + }, + ], }), ) .expect(400); diff --git a/packages/api/src/routers/external-api/v2/dashboards.ts b/packages/api/src/routers/external-api/v2/dashboards.ts index 77d3b31361..5de80d7b96 100644 --- a/packages/api/src/routers/external-api/v2/dashboards.ts +++ b/packages/api/src/routers/external-api/v2/dashboards.ts @@ -57,11 +57,12 @@ const EXTERNAL_DASHBOARD_PROJECTION = { * SavedFilterValue: * description: > * A single saved dashboard filter selection. Either a rendered SQL - * condition, or a selection addressed by the name of the dashboard - * variable it belongs to. + * condition, a selection addressed by the name of the dashboard + * variable it belongs to, or the conditions of an ADHOC filter. * oneOf: * - $ref: '#/components/schemas/SqlSavedFilterValue' * - $ref: '#/components/schemas/VariableSavedFilterValue' + * - $ref: '#/components/schemas/AdhocSavedFilterValue' * SqlSavedFilterValue: * type: object * required: [condition] @@ -102,6 +103,54 @@ const EXTERNAL_DASHBOARD_PROJECTION = { * type: string * maxLength: 10000 * example: ["hdx-oss-dev-api"] + * AdhocSavedFilterValue: + * type: object + * required: [type, name, conditions] + * properties: + * type: + * type: string + * enum: [adhoc] + * description: Filter type. + * example: "adhoc" + * name: + * type: string + * minLength: 1 + * maxLength: 1024 + * description: The variableName of the ADHOC filter these conditions belong to. + * example: "conditions" + * conditions: + * type: array + * maxItems: 100 + * description: The filter's conditions. + * items: + * $ref: '#/components/schemas/AdhocFilterCondition' + * AdhocFilterCondition: + * type: object + * description: > + * One "key operator value" condition. Conditions are AND'd, except that + * IN (sql) and =| (promql) conditions sharing a key are OR'd into one. + * required: [key, operator, value] + * properties: + * key: + * type: string + * minLength: 1 + * maxLength: 10000 + * description: A SQL expression (column or map access) for sql filters, or a label name for promql filters. + * example: "ServiceName" + * operator: + * type: string + * enum: ["=", "!=", "=~", "!~", "LIKE", "NOT LIKE", "IN", "=|"] + * description: > + * Comparison operator. =~ and !~ match a regular expression. LIKE, + * NOT LIKE, and IN apply to sql filters only, and =| to promql + * filters only; a condition its filter's sourceType can't express + * is not applied. + * example: "=" + * value: + * type: string + * maxLength: 10000 + * description: Value, pattern, or regular expression to compare against. + * example: "api" * MetricDataType: * type: string * enum: [sum, gauge, histogram, summary, exponential histogram] @@ -2368,12 +2417,14 @@ const EXTERNAL_DASHBOARD_PROJECTION = { * - $ref: '#/components/schemas/QueryExpressionFilterInput' * - $ref: '#/components/schemas/StaticListFilterInput' * - $ref: '#/components/schemas/PrometheusLabelFilterInput' + * - $ref: '#/components/schemas/AdhocFilterInput' * discriminator: * propertyName: type * mapping: * QUERY_EXPRESSION: '#/components/schemas/QueryExpressionFilterInput' * STATIC_LIST: '#/components/schemas/StaticListFilterInput' * PROMETHEUS_LABEL: '#/components/schemas/PrometheusLabelFilterInput' + * ADHOC: '#/components/schemas/AdhocFilterInput' * * QueryExpressionFilterInput: * type: object @@ -2684,6 +2735,92 @@ const EXTERNAL_DASHBOARD_PROJECTION = { * behavior. * example: 1 * + * AdhocFilterInput: + * type: object + * description: | + * A filter holding free-form "key operator value" conditions, with keys + * and values suggested from its sources: columns and map keys for sql + * sources, labels for promql sources. Its conditions are always + * broadcast to the tiles it applies to: as WHERE conditions on builder + * tiles and raw sql tiles using the $__filters macro, or as label + * matchers on PromQL tiles. Its conditions are saved in + * savedFilterValues as an "adhoc" entry. Ad hoc filters are a preview + * feature. + * required: + * - type + * - name + * - sourceType + * - sources + * properties: + * type: + * type: string + * enum: [ADHOC] + * description: Discriminator. Must be "ADHOC". + * example: "ADHOC" + * name: + * type: string + * minLength: 1 + * description: Display name for the dashboard filter + * example: "Conditions" + * sourceType: + * type: string + * enum: [sql, promql] + * description: | + * The kind of sources the filter reads keys from and applies to: Log, + * Trace, or Session sources for sql, PromQL sources for promql. + * example: "sql" + * sources: + * type: array + * minItems: 1 + * items: + * type: string + * description: IDs of the sources whose keys and values the filter suggests, all of sourceType. + * example: ["65f5e4a3b9e77c001a111111"] + * where: + * type: string + * description: | + * Optional condition narrowing the rows keys and values are suggested + * from. Never applied to tiles. May reference the dashboard's + * variables. sql filters only: rejected when sourceType is promql. + * example: "ServiceName:api" + * whereLanguage: + * type: string + * enum: [sql, lucene] + * description: Language of the where condition. Rejected when sourceType is promql. + * example: "lucene" + * appliesToSourceIds: + * type: array + * items: + * type: string + * description: | + * IDs of the sources whose tiles the conditions apply to, all of + * sourceType. Omit or send an empty array to apply to the tiles of + * the filter's sources. + * example: ["65f5e4a3b9e77c001a111111"] + * isBroadcastEnabled: + * type: boolean + * enum: [true] + * default: true + * description: Must be true or omitted. + * example: true + * isVariableEnabled: + * type: boolean + * enum: [true] + * default: true + * description: Must be true or omitted. + * example: true + * variableName: + * type: string + * maxLength: 64 + * pattern: '^[a-zA-Z][a-zA-Z0-9_]*$' + * description: | + * Unique name the filter's conditions are saved under, in + * savedFilterValues and the dashboard URL. Tiles can't reference it + * as a variable. Defaults to the display name with whitespace + * replaced by underscores and remaining illegal characters removed. + * Must be unique across a dashboard's variable-enabled filters. + * example: "conditions" + * * Filter: * allOf: * - $ref: '#/components/schemas/FilterInput' @@ -3603,6 +3740,7 @@ router.put( tiles, filters, existingTiles: existingDashboard.tiles ?? [], + existingFilters: existingDashboard.filters ?? [], containers: effectiveContainers, }); if (validationError) { diff --git a/packages/api/src/routers/external-api/v2/utils/__tests__/dashboards.test.ts b/packages/api/src/routers/external-api/v2/utils/__tests__/dashboards.test.ts index dc8882183e..dec4319981 100644 --- a/packages/api/src/routers/external-api/v2/utils/__tests__/dashboards.test.ts +++ b/packages/api/src/routers/external-api/v2/utils/__tests__/dashboards.test.ts @@ -508,6 +508,43 @@ describe('convertToExternalDashboard orphan-ref heal', () => { expect(hidden.filters?.map(f => f.id)).toEqual(['static-filter']); }); + it('hides ad hoc filters and their saved conditions only when asked', () => { + const adhoc = { + id: 'adhoc-filter', + type: 'ADHOC' as const, + name: 'Conditions', + sourceType: 'sql' as const, + sources: [new mongoose.Types.ObjectId().toString()], + isBroadcastEnabled: true as const, + isVariableEnabled: true as const, + variableName: 'conds', + }; + const variableValue = { + type: 'variable' as const, + name: 'env', + values: ['prod'], + }; + const adhocValue = { + type: 'adhoc' as const, + name: 'conds', + conditions: [ + { key: 'ServiceName', operator: '=' as const, value: 'api' }, + ], + }; + const doc = makeDoc({ + filters: [adhoc], + savedFilterValues: [variableValue, adhocValue], + }); + + const shown = convertToExternalDashboard(doc); + expect(shown.filters).toEqual([adhoc]); + expect(shown.savedFilterValues).toEqual([variableValue, adhocValue]); + + const hidden = convertToExternalDashboard(doc, { hideAdhoc: true }); + expect(hidden.filters).toEqual([]); + expect(hidden.savedFilterValues).toEqual([variableValue]); + }); + // seriesLimit is three-state (matching the internal schema): omitted = // default cap, 0 = unlimited, positive N = top-N. All three must survive a // GET->PUT round-trip: 0 and N pass through, null maps to absent (the diff --git a/packages/api/src/routers/external-api/v2/utils/dashboards.ts b/packages/api/src/routers/external-api/v2/utils/dashboards.ts index 69e143e950..5820842f60 100644 --- a/packages/api/src/routers/external-api/v2/utils/dashboards.ts +++ b/packages/api/src/routers/external-api/v2/utils/dashboards.ts @@ -15,7 +15,7 @@ import { } from '@hyperdx/common-utils/dist/dashboardValidation'; import { isPrometheusLabelFilter, - isStaticListFilter, + isQueryExpressionFilter, } from '@hyperdx/common-utils/dist/filters'; import { isBuilderSavedChartConfig, @@ -30,12 +30,12 @@ import { DASHBOARD_MAX_CONTAINERS, DashboardContainer, DashboardContainerSchema, + DashboardFilter, DisplayType, HeatmapMode, isLogSource, isOnClickDashboardById, isOnClickSearchById, - isPromqlSource, isTraceSource, NumberTileColorCondition, NumberTileColorConditionSchema, @@ -43,6 +43,7 @@ import { RawSqlSavedChartConfig, resolveChartPaletteToken, SavedChartConfig, + SourceKind, } from '@hyperdx/common-utils/dist/types'; import { SearchConditionLanguageSchema as whereLanguageSchema } from '@hyperdx/common-utils/dist/types'; import { pick } from 'lodash'; @@ -55,12 +56,17 @@ import { getConnectionsByTeam } from '@/controllers/connection'; import { getSources } from '@/controllers/sources'; import Dashboard, { DashboardDocument } from '@/models/dashboard'; import { - getExternalCompatibleFilters, - getExternalCompatibleFilterValues, + getMcpCompatibleFilters, + getMcpCompatibleFilterValues, translateExternalChartToTileConfig, translateExternalFilterToFilter, translateFilterToExternalFilter, } from '@/utils/externalApi'; +import { + getChangedFilterSourceRefs, + getFilterSourceRefError, + getInvalidSourceIds, +} from '@/utils/filterSources'; import logger from '@/utils/logger'; import { ExternalDashboardBuilderTileConfig, @@ -773,10 +779,13 @@ export function convertToExternalDashboard( dashboard: DashboardDocument, { hidePromql = false, + hideAdhoc = false, }: { // MCP hides PromQL tiles and PROMETHEUS_LABEL filters while PromQL is // disabled on the deployment. hidePromql?: boolean; + // MCP doesn't expose ad hoc filters or their saved conditions yet. + hideAdhoc?: boolean; } = {}, ): ExternalDashboard { const containers = dashboard.containers ?? []; @@ -798,14 +807,17 @@ export function convertToExternalDashboard( convertTileToExternalChart(tile, containerById, dashboardId), ), tags: dashboard.tags || [], - filters: getExternalCompatibleFilters(dashboard.filters) + filters: (hideAdhoc + ? getMcpCompatibleFilters(dashboard.filters) + : (dashboard.filters ?? []) + ) .filter(filter => !(hidePromql && isPrometheusLabelFilter(filter))) .map(translateFilterToExternalFilter), savedQuery: dashboard.savedQuery ?? null, savedQueryLanguage: dashboard.savedQueryLanguage ?? null, - savedFilterValues: getExternalCompatibleFilterValues( - dashboard.savedFilterValues, - ), + savedFilterValues: hideAdhoc + ? getMcpCompatibleFilterValues(dashboard.savedFilterValues) + : (dashboard.savedFilterValues ?? []), // Mongoose persists missing arrays as []. Only emit containers when // the user actually saved one or more, so dashboards without the // organization layer round-trip with the field absent. @@ -1307,13 +1319,10 @@ function getMissingSources( } } - if (filters?.length) { - for (const filter of filters) { - // Every variant but the static one names a source. - if (!isStaticListFilter(filter)) { - sourceIds.add(filter.sourceId); - } - } + for (const filter of filters ?? []) { + // PROMETHEUS_LABEL and ADHOC sources are checked with the stored filter's + // references skipped; see `validateDashboardTiles`. + if (isQueryExpressionFilter(filter)) sourceIds.add(filter.sourceId); } const existingSourceIds = new Set( @@ -1354,25 +1363,6 @@ function getHeatmapTilesWithIncompatibleSources( return { distribution: [...distribution], series: [...series] }; } -/** - * Returns an error message string if any of the referenced source IDs is not - * a valid PromQL source, or null if all of them are. `subject` names what - * referenced them, e.g. "PromQL tiles". - */ -function getNonPromqlSourceError( - sources: SourceForValidation[], - referencedSourceIds: string[], - subject: string, -): string | null { - const sourceById = new Map(sources.map(s => [s._id.toString(), s])); - const invalid = [...new Set(referencedSourceIds)].filter(id => { - const source = sourceById.get(id); - return source === undefined || !isPromqlSource(source); - }); - if (invalid.length === 0) return null; - return `${subject} require a PromQL source. The following source IDs are not PromQL sources: ${invalid.join(', ')}`; -} - /** * Returns source IDs referenced by formula tiles that exist but are not * formula-capable. Formulas render on metric and log/trace sources; other @@ -1731,6 +1721,8 @@ export type TileValidationContext = { filters?: (ExternalDashboardFilter | ExternalDashboardFilterWithId)[]; /** Existing internal tiles for scoping heatmap change detection (update paths). */ existingTiles?: DashboardDocument['tiles']; + /** Existing filters, whose source references aren't re-checked (update paths). */ + existingFilters?: DashboardFilter[]; /** Container set to validate tile containerId/tabId refs against. */ containers: DashboardContainer[]; }; @@ -1747,7 +1739,8 @@ export type TileValidationContext = { export async function validateDashboardTiles( ctx: TileValidationContext, ): Promise { - const { teamId, tiles, filters, existingTiles, containers } = ctx; + const { teamId, tiles, filters, existingTiles, existingFilters, containers } = + ctx; // Container/tab ref resolution. const tileRefIssues = collectTileContainerRefIssues(containers, tiles); @@ -1815,22 +1808,33 @@ export async function validateDashboardTiles( return `Tiles with formulas require a Metric, Log, or Trace source. The following source IDs are not formula-capable: ${formulaIncompatibleSources.join(', ')}`; } - // PromQL tiles and PROMETHEUS_LABEL filters query through their source's - // connection and TimeSeries table, which only a PromQL source carries. - const promqlTileError = getNonPromqlSourceError( + // PromQL tiles query through their source's connection and TimeSeries + // table, which only a PromQL source carries. + const nonPromqlTileSourceIds = getInvalidSourceIds( + getChangedPromqlTileSourceIds(tiles, existingTiles).map(id => ({ + id, + kinds: [SourceKind.Promql], + })), sources, - getChangedPromqlTileSourceIds(tiles, existingTiles), - 'PromQL tiles', ); - if (promqlTileError != null) return promqlTileError; + if (nonPromqlTileSourceIds.length > 0) { + return `PromQL tiles require a PromQL source. The following source IDs are not PromQL sources: ${nonPromqlTileSourceIds.join(', ')}`; + } - const promqlLabelFilters = (filters ?? []).filter(isPrometheusLabelFilter); - const promqlLabelFilterError = getNonPromqlSourceError( + // PROMETHEUS_LABEL and ADHOC filters need live sources of the right kind. As + // in the internal API, references the stored filter already made are not + // re-checked, so a since-deleted source doesn't block unrelated saves. + const filterSourceError = getFilterSourceRefError( + getChangedFilterSourceRefs( + // Create bodies carry no filter IDs; an empty one matches no stored filter. + (filters ?? []).map(filter => + translateExternalFilterToFilter({ id: '', ...filter }), + ), + existingFilters, + ), sources, - promqlLabelFilters.map(f => f.sourceId), - 'PROMETHEUS_LABEL filters', ); - if (promqlLabelFilterError != null) return promqlLabelFilterError; + if (filterSourceError != null) return filterSourceError; if (missingOnClickDashboards.length > 0) { return `Could not find the following onClick dashboard IDs: ${missingOnClickDashboards.join(', ')}`; diff --git a/packages/api/src/utils/__tests__/filterSources.test.ts b/packages/api/src/utils/__tests__/filterSources.test.ts new file mode 100644 index 0000000000..10d8d2a4b4 --- /dev/null +++ b/packages/api/src/utils/__tests__/filterSources.test.ts @@ -0,0 +1,108 @@ +import { DashboardFilter, SourceKind } from '@hyperdx/common-utils/dist/types'; + +import { + getChangedFilterSourceRefs, + getFilterSourceRefError, +} from '@/utils/filterSources'; + +const adhoc = ( + overrides: Partial> = {}, +): DashboardFilter => ({ + id: 'a1', + type: 'ADHOC', + name: 'Conditions', + sourceType: 'sql', + sources: ['logs'], + isBroadcastEnabled: true, + isVariableEnabled: true, + ...overrides, +}); + +const label: DashboardFilter = { + id: 'l1', + type: 'PROMETHEUS_LABEL', + name: 'Job', + source: 'prom', + label: 'job', + isBroadcastEnabled: false, + isVariableEnabled: true, +}; + +const sql = [SourceKind.Log, SourceKind.Trace, SourceKind.Session]; + +describe('getChangedFilterSourceRefs', () => { + it('returns every reference without stored filters', () => { + expect( + getChangedFilterSourceRefs([ + adhoc({ appliesToSourceIds: ['traces'] }), + label, + ]), + ).toEqual([ + { id: 'logs', kinds: sql }, + { id: 'traces', kinds: sql }, + { id: 'prom', kinds: [SourceKind.Promql] }, + ]); + }); + + it('skips references the stored filter with the same ID already made', () => { + expect( + getChangedFilterSourceRefs( + [adhoc({ sources: ['logs', 'traces'] }), label], + [adhoc(), label], + ), + ).toEqual([{ id: 'traces', kinds: sql }]); + }); + + it('re-checks a reference whose required kinds changed', () => { + expect( + getChangedFilterSourceRefs([adhoc({ sourceType: 'promql' })], [adhoc()]), + ).toEqual([{ id: 'logs', kinds: [SourceKind.Promql] }]); + }); + + it('checks a filter without an ID, or with a new one', () => { + const { id: _id, ...withoutId } = adhoc(); + expect( + getChangedFilterSourceRefs( + [withoutId as DashboardFilter, adhoc({ id: 'new' })], + [adhoc()], + ), + ).toEqual([ + { id: 'logs', kinds: sql }, + { id: 'logs', kinds: sql }, + ]); + }); +}); + +describe('getFilterSourceRefError', () => { + const sources = [ + { _id: 'logs', kind: SourceKind.Log }, + { _id: 'prom', kind: SourceKind.Promql }, + ]; + + it('accepts references to existing sources of the right kind', () => { + expect( + getFilterSourceRefError( + [ + { id: 'logs', kinds: sql }, + { id: 'prom', kinds: [SourceKind.Promql] }, + ], + sources, + ), + ).toBeNull(); + }); + + it('names missing sources and sources of the wrong kind once each', () => { + expect( + getFilterSourceRefError( + [ + { id: 'prom', kinds: sql }, + { id: 'gone', kinds: sql }, + { id: 'gone', kinds: sql }, + ], + sources, + ), + ).toBe( + 'The following filter source IDs do not exist or are not a kind of source the filter supports: prom, gone', + ); + }); +}); diff --git a/packages/api/src/utils/externalApi.ts b/packages/api/src/utils/externalApi.ts index 187abd7a24..9e2d6ff488 100644 --- a/packages/api/src/utils/externalApi.ts +++ b/packages/api/src/utils/externalApi.ts @@ -240,32 +240,22 @@ export function translateExternalChartToTileConfig( }; } -/** Filter types the external API and MCP don't expose yet. */ -type InternalOnlyDashboardFilter = Extract; -export type ExternalCompatibleDashboardFilter = Exclude< - DashboardFilter, - InternalOnlyDashboardFilter ->; - -/** The filters the external API and MCP may read. */ -export function getExternalCompatibleFilters( +/** The filters MCP may read; it doesn't expose ad hoc filters yet. */ +export function getMcpCompatibleFilters( filters: DashboardFilter[] | undefined, -): ExternalCompatibleDashboardFilter[] { - return (filters ?? []).filter( - (filter): filter is ExternalCompatibleDashboardFilter => - !isAdhocFilter(filter), - ); +): DashboardFilter[] { + return (filters ?? []).filter(filter => !isAdhocFilter(filter)); } -/** Saved filter values the external API and MCP may read. */ -export function getExternalCompatibleFilterValues( +/** Saved filter values MCP may read; it doesn't expose ad hoc filters yet. */ +export function getMcpCompatibleFilterValues( values: IDashboard['savedFilterValues'], ): NonNullable { return (values ?? []).filter(value => !isAdhocFilterValue(value)); } export function translateFilterToExternalFilter( - filter: ExternalCompatibleDashboardFilter, + filter: DashboardFilter, ): ExternalDashboardFilterWithId { // Omitted from responses unless the filter is required, so a GET response // can be PUT back verbatim. @@ -277,6 +267,12 @@ export function translateFilterToExternalFilter( case 'STATIC_LIST': return omit(filter, ...omittedKeys); + case 'ADHOC': + // Only sql filters read `where`; the internal API may have stored one. + return filter.sourceType === 'promql' + ? omit(filter, 'where', 'whereLanguage') + : filter; + case 'PROMETHEUS_LABEL': return { ...omit(filter, 'source', ...omittedKeys), @@ -309,6 +305,7 @@ export function translateExternalFilterToFilter( ): DashboardFilter { switch (filter.type) { case 'STATIC_LIST': + case 'ADHOC': return filter; case 'PROMETHEUS_LABEL': diff --git a/packages/api/src/utils/filterSources.ts b/packages/api/src/utils/filterSources.ts new file mode 100644 index 0000000000..2cc2df23fa --- /dev/null +++ b/packages/api/src/utils/filterSources.ts @@ -0,0 +1,80 @@ +import { + getAdhocFilterSourceKinds, + isAdhocFilter, + isPrometheusLabelFilter, +} from '@hyperdx/common-utils/dist/filters'; +import { DashboardFilter, SourceKind } from '@hyperdx/common-utils/dist/types'; + +/** A filter as stored or as sent on create, when it has no ID yet. */ +type FilterWithOptionalId = T extends unknown + ? Omit & { id?: string } + : never; + +/** A source a reference points at, and the kinds it must be. */ +export type SourceRef = { id: string; kinds: SourceKind[] }; + +/** The sources a filter reads from, each with the kinds it must be. */ +function getFilterSourceRefs(filter: FilterWithOptionalId): SourceRef[] { + if (isPrometheusLabelFilter(filter)) { + return [{ id: filter.source, kinds: [SourceKind.Promql] }]; + } + if (isAdhocFilter(filter)) { + const kinds = getAdhocFilterSourceKinds(filter.sourceType); + return [...filter.sources, ...(filter.appliesToSourceIds ?? [])].map( + id => ({ id, kinds }), + ); + } + return []; +} + +/** + * The source references in `filters` that the stored filter with the same ID + * didn't already make, so a since-deleted source doesn't block unrelated saves. + */ +export function getChangedFilterSourceRefs( + filters: FilterWithOptionalId[] = [], + existingFilters: DashboardFilter[] = [], +): SourceRef[] { + const refKey = ({ id, kinds }: SourceRef) => `${id}:${kinds.join(',')}`; + const existingRefsById = new Map( + existingFilters.map(filter => [ + filter.id, + new Set(getFilterSourceRefs(filter).map(refKey)), + ]), + ); + return filters.flatMap(filter => + getFilterSourceRefs(filter).filter( + ref => + filter.id === undefined || + !existingRefsById.get(filter.id)?.has(refKey(ref)), + ), + ); +} + +/** The referenced source IDs that are missing or of the wrong kind, once each. */ +export function getInvalidSourceIds( + refs: SourceRef[], + sources: { _id: { toString(): string }; kind: SourceKind }[], +): string[] { + const kindById = new Map(sources.map(s => [s._id.toString(), s.kind])); + const invalid = refs + .filter(({ id, kinds }) => { + const kind = kindById.get(id); + return kind === undefined || !kinds.includes(kind); + }) + .map(({ id }) => id); + return [...new Set(invalid)]; +} + +/** + * An error naming the referenced sources that are missing or of the wrong + * kind, or null when there are none. + */ +export function getFilterSourceRefError( + refs: SourceRef[], + sources: Parameters[1], +): string | null { + const invalid = getInvalidSourceIds(refs, sources); + if (invalid.length === 0) return null; + return `The following filter source IDs do not exist or are not a kind of source the filter supports: ${invalid.join(', ')}`; +} diff --git a/packages/api/src/utils/zod.ts b/packages/api/src/utils/zod.ts index d14d34849d..5a27dccf2b 100644 --- a/packages/api/src/utils/zod.ts +++ b/packages/api/src/utils/zod.ts @@ -1,6 +1,8 @@ import { validateChartConfigFormulas } from '@hyperdx/common-utils/dist/dashboardValidation'; import { addDuplicateTileIdIssues, + AdhocDashboardFilterSchema, + AdhocFilterValueSchema, AggregateFunctionSchema, AlertChartConfigSchema, alertDisplayNameSchema, @@ -189,12 +191,21 @@ const externalPromqlLabelFilterShape = PromqlLabelDashboardFilterSchema.omit({ isVariableEnabled: z.literal(true).default(true), }); +// Field names match the stored filter, so it needs no translation. +const externalAdhocFilterShape = AdhocDashboardFilterSchema.extend({ + sources: z.array(objectIdSchema).min(1), + appliesToSourceIds: z.array(objectIdSchema).optional(), + isBroadcastEnabled: z.literal(true).default(true), + isVariableEnabled: z.literal(true).default(true), +}); + export const externalDashboardFilterSchemaWithId = z.discriminatedUnion( 'type', [ externalQueryExpressionFilterShape.strict(), externalStaticListFilterShape.strict(), externalPromqlLabelFilterShape.strict(), + externalAdhocFilterShape.strict(), ], ); @@ -211,6 +222,7 @@ export const externalDashboardFilterSchema = z.discriminatedUnion('type', [ externalQueryExpressionFilterShape.omit({ id: true }).strict(), externalStaticListFilterShape.omit({ id: true }).strict(), externalPromqlLabelFilterShape.omit({ id: true }).strict(), + externalAdhocFilterShape.omit({ id: true }).strict(), ]); export type ExternalDashboardFilter = z.infer< @@ -218,8 +230,9 @@ export type ExternalDashboardFilter = z.infer< >; /** - * One entry in a dashboard's `savedFilterValues`: either a rendered SQL - * predicate, or a selection addressed by the dashboard variable it belongs to. + * One entry in a dashboard's `savedFilterValues`: a rendered SQL predicate, a + * selection addressed by the dashboard variable it belongs to, or an ad hoc + * filter's conditions addressed by its variable name. */ export const externalSqlSavedFilterValueSchema = z.object({ type: z.literal('sql').optional().default('sql'), @@ -232,6 +245,7 @@ export const externalVariableSavedFilterValueSchema = export const externalDashboardSavedFilterValueSchema = z.union([ externalSqlSavedFilterValueSchema, externalVariableSavedFilterValueSchema, + AdhocFilterValueSchema.strict(), ]); // ================================ diff --git a/packages/common-utils/src/__tests__/dashboardValidation.test.ts b/packages/common-utils/src/__tests__/dashboardValidation.test.ts index ad823b40be..fc36b51057 100644 --- a/packages/common-utils/src/__tests__/dashboardValidation.test.ts +++ b/packages/common-utils/src/__tests__/dashboardValidation.test.ts @@ -284,6 +284,10 @@ describe('validateDashboardFilterModes', () => { type GatingFilter = ModeFilter & { variableName?: string; appliesToSourceIds?: string[]; + type?: string; + sourceType?: string; + where?: string; + whereLanguage?: string; }; const collectGatingIssues = ( @@ -304,6 +308,26 @@ describe('validateDashboardFilterFieldGating', () => { expect(collectGatingIssues([])).toEqual([]); }); + it('rejects where and whereLanguage on a PromQL ad hoc filter only', () => { + const adhoc = (sourceType: string) => ({ + name: sourceType, + type: 'ADHOC', + sourceType, + isBroadcastEnabled: true, + where: 'Namespace:api', + whereLanguage: 'lucene', + }); + const issues = collectGatingIssues([adhoc('sql'), adhoc('promql')]); + + expect(issues.map(issue => issue.path)).toEqual([ + ['filters', 1, 'where'], + ['filters', 1, 'whereLanguage'], + ]); + expect(issues[0].message).toBe( + 'Filter "promql" sets where, which only ad hoc filters with sourceType "sql" use; drop where', + ); + }); + it('accepts a filter that carries neither field', () => { expect(collectGatingIssues([{ name: 'Service' }])).toEqual([]); }); diff --git a/packages/common-utils/src/dashboardValidation.ts b/packages/common-utils/src/dashboardValidation.ts index 8a88bf999e..65cc31ed0f 100644 --- a/packages/common-utils/src/dashboardValidation.ts +++ b/packages/common-utils/src/dashboardValidation.ts @@ -30,6 +30,10 @@ type FilterForModeValidation = { type FilterForGatingValidation = FilterForModeValidation & { variableName?: string; appliesToSourceIds?: string[]; + type?: string; + sourceType?: string; + where?: string; + whereLanguage?: string; }; /** @@ -238,6 +242,8 @@ export function validateDashboardFilterModes( * Issues raised: * - `variableName` on a filter that is not variable-enabled (path `[i].variableName`). * - A non-empty `appliesToSourceIds` on a filter that does not broadcast (path `[i].appliesToSourceIds`). + * - `where` or `whereLanguage` on a PromQL ad hoc filter, which only SQL ad hoc + * filters read (path `[i].where` / `.whereLanguage`). */ export function validateDashboardFilterFieldGating< T extends FilterForGatingValidation, @@ -268,6 +274,17 @@ export function validateDashboardFilterFieldGating< path: [...filtersPath, filterIdx, 'appliesToSourceIds'], }); } + + if (filter.type === 'ADHOC' && filter.sourceType === 'promql') { + for (const field of ['where', 'whereLanguage'] as const) { + if (filter[field] === undefined) continue; + ctx.addIssue({ + code: z.ZodIssueCode.custom, + message: `Filter "${filter.name}" sets ${field}, which only ad hoc filters with sourceType "sql" use; drop ${field}`, + path: [...filtersPath, filterIdx, field], + }); + } + } }); }