diff --git a/.changeset/sql-query-dashboard-filter-app.md b/.changeset/sql-query-dashboard-filter-app.md new file mode 100644 index 000000000..9c9ec29a0 --- /dev/null +++ b/.changeset/sql-query-dashboard-filter-app.md @@ -0,0 +1,8 @@ +--- +'@hyperdx/common-utils': patch +'@hyperdx/app': patch +--- + +feat: create SQL query dashboard filters and fill their dropdowns + +Dashboard filters can now take their values from a ClickHouse SQL query. The filter editor offers a "SQL query values" type with a connection, a SQL editor, and a preview of the values. The dropdown lists the query's first column in the order it returns, up to 10,000 values, and the query can use the time macros and other filters' variables. diff --git a/packages/app/src/DBDashboardPage.tsx b/packages/app/src/DBDashboardPage.tsx index f5000d1a7..c3172241e 100644 --- a/packages/app/src/DBDashboardPage.tsx +++ b/packages/app/src/DBDashboardPage.tsx @@ -3510,6 +3510,7 @@ function DBDashboardPage({ showVariableOptions showRequiredFilterOptions variables={variables} + dateRange={searchedTimeRange} /> )} diff --git a/packages/app/src/DashboardFilters.tsx b/packages/app/src/DashboardFilters.tsx index 5c15cdf9a..ee1109412 100644 --- a/packages/app/src/DashboardFilters.tsx +++ b/packages/app/src/DashboardFilters.tsx @@ -9,6 +9,7 @@ import { isFilterRequired, isFilterVariableEnabled, isQueryExpressionFilter, + isSqlQueryFilter, isStaticListFilter, } from '@hyperdx/common-utils/dist/filters'; import { @@ -141,8 +142,9 @@ const DashboardFilterSelect = ({ // so a completed/empty/failed query stays interactive and the user can // still clear or adjust the selection. loading={isLoading} - // A static list renders in the order its author wrote it. - sort={!isStaticListFilter(filter)} + // A static list renders in the order its author wrote it, and a SQL + // query filter in the order its query returns. + sort={!isStaticListFilter(filter) && !isSqlQueryFilter(filter)} isMultiSelect={!isFilterSingleSelect(filter)} onChange={onChange} data-testid={`dashboard-filter-select-${filter.name}`} @@ -241,11 +243,16 @@ const DashboardFilters = ({ const isLoadingValues = queriedFilterValues ? queriedFilterValues.isLoading : isFetching; - // Only a queried dropdown has a values query that can await a - // variable's selection. + // Only queried and SQL query dropdowns have a values query that can + // await a variable's selection. const pendingVariables = isQueryExpressionFilter(filter) ? getPendingFilterValuesVariables(filter, variables) - : undefined; + : isSqlQueryFilter(filter) + ? getPendingFilterValuesVariables( + { where: filter.sqlTemplate, whereLanguage: 'sql' }, + variables, + ) + : undefined; return ( void; onClose: () => void; onCancel: () => void; @@ -97,6 +101,7 @@ export const DashboardFilterEditForm = ({ showVariableOptions, showRequiredFilterOptions, variables, + dateRange, onSave, onClose, onCancel, @@ -249,6 +254,14 @@ export const DashboardFilterEditForm = ({ otherFilters={otherFilters} /> + ) : formFilterType === 'SQL_QUERY' ? ( + + + ) : formFilterType === 'PROMETHEUS_LABEL' ? ( ([ + ...SOURCE_DEPENDENT_MACROS, + ...INTERVAL_MACROS, +]); + +const PLACEHOLDER_SQL = `SELECT DISTINCT env +FROM default.environments +WHERE $__filter(cloud, $cloud) +ORDER BY env`; + +interface SqlQueryFilterEditFormProps { + control: FilterFormControl; + /** Filters other than the one being edited, used to keep variable names unique. */ + otherFilters: DashboardFilter[]; + /** The range the preview runs over. Defaults to the last hour. */ + dateRange?: [Date, Date]; +} + +/** Form describing a dashboard filter whose values come from a SQL query */ +export const SqlQueryFilterEditForm = ({ + control, + otherFilters, + dateRange, +}: SqlQueryFilterEditFormProps) => { + const { errors } = useFormState({ control }); + const variables = useChartVariables(); + const completions = useMemo( + () => [ + ...MACRO_SUGGESTIONS.filter( + // `sourceTable(gauge)` etc. are per-metric variants of `sourceTable`. + macro => !UNSUPPORTED_MACROS.has(macro.name.split('(')[0]), + ).map(toMacroCompletion), + ...buildSqlVariableCompletions(variables), + ], + [variables], + ); + + const [connection, sqlTemplate] = useWatch({ + control, + name: ['connection', 'sqlTemplate'], + }); + // What the preview last ran, so editing doesn't re-query on every keystroke. + const [previewed, setPreviewed] = useState<{ + connection: string; + sqlTemplate: string; + }>(); + const canPreview = !!connection && !!sqlTemplate?.trim(); + const runPreview = () => { + if (canPreview) setPreviewed({ connection, sqlTemplate }); + }; + + return ( + <> + +
+ +
+
+ + + + { + if (typeof value !== 'string' || !value.trim()) { + return 'This field is required'; + } + // `variables` leaves out this filter's own, so a reference to + // it is rejected as unknown. + return ( + validateSqlQueryFilterTemplate(value, variables ?? []) ?? true + ); + }, + }} + height="150px" + enableLineWrapping + placeholder={PLACEHOLDER_SQL} + additionalCompletions={completions} + onSubmit={runPreview} + /> + + + + {previewed && ( + + )} + + + + + + ); +}; + +const SqlQueryFilterPreview = ({ + connection, + sqlTemplate, + dateRange, + variables, +}: { + connection: string; + sqlTemplate: string; + dateRange?: [Date, Date]; + variables: ReturnType; +}) => { + const lastHour = useMemo( + (): [Date, Date] => [new Date(NOW - 60 * 60 * 1000), new Date(NOW)], + [], + ); + const filters = useMemo( + (): SqlQueryDashboardFilter[] => [ + { + id: 'preview', + type: 'SQL_QUERY', + name: 'Preview', + connection, + sqlTemplate, + isBroadcastEnabled: false, + isVariableEnabled: true, + }, + ], + [connection, sqlTemplate], + ); + const { data, filterErrorMessages, isLoading, isFetching, isError } = + useSqlQueryFilterValues({ + filters, + dateRange: dateRange ?? lastHour, + variables, + }); + const values = data.get('preview')?.values ?? []; + + // While a re-run loads, the previous query's values stand in as placeholder + // data, so isLoading alone would show them as this query's results. + if (isLoading || isFetching) { + return ( + + + Running query + + ); + } + + if (isError) { + return ( + } + data-testid="filter-sql-preview" + > + {filterErrorMessages.get('preview') ?? 'The query failed.'} + + ); + } + + if (values.length === 0) { + return ( +
+ +
+ ); + } + + const notShown = values.length - PREVIEW_VALUES_SHOWN; + return ( + + + {values.length} {values.length === 1 ? 'value' : 'values'} + + + {values.slice(0, PREVIEW_VALUES_SHOWN).map((value, index) => ( + + {index > 0 && ', '} + + {truncateText(value, PREVIEW_VALUE_MAX_LENGTH, '…')} + + + ))} + {notShown > 0 && `, and ${notShown} more`} + + + ); +}; diff --git a/packages/app/src/components/DashboardFiltersModal/__tests__/SqlQueryFilterEditForm.test.tsx b/packages/app/src/components/DashboardFiltersModal/__tests__/SqlQueryFilterEditForm.test.tsx new file mode 100644 index 000000000..8c69569e5 --- /dev/null +++ b/packages/app/src/components/DashboardFiltersModal/__tests__/SqlQueryFilterEditForm.test.tsx @@ -0,0 +1,226 @@ +import { RegisterOptions, useForm } from 'react-hook-form'; +import { screen } from '@testing-library/react'; +import userEvent from '@testing-library/user-event'; + +import { + FilterFormControl, + FilterFormValues, + toFormValues, +} from '@/components/DashboardFiltersModal/filterFormState'; +import { SqlQueryFilterEditForm } from '@/components/DashboardFiltersModal/SqlQueryFilterEditForm'; +import { SqlVariablesProvider } from '@/components/SQLEditor/variableCompletions'; + +const mockUseSqlQueryFilterValues = jest.fn(); +const mockCompletionLabels: string[] = []; + +jest.mock('@/hooks/useSqlQueryFilterValues', () => ({ + useSqlQueryFilterValues: (...args: unknown[]) => + mockUseSqlQueryFilterValues(...args), +})); +// The real selector fetches connections, and the real editor is CodeMirror; +// plain inputs bound to the same fields are enough to drive the preview. +jest.mock('@/components/ConnectionSelect', () => ({ + ConnectionSelectControlled: ({ + control, + name, + }: { + control: FilterFormControl; + name: 'connection'; + }) => , +})); +jest.mock('@/components/SQLEditor/SQLEditor', () => ({ + SQLEditorControlled: ({ + control, + name, + rules, + additionalCompletions, + }: { + control: FilterFormControl; + name: 'sqlTemplate'; + rules?: RegisterOptions; + additionalCompletions?: { label: string }[]; + }) => { + mockCompletionLabels.splice( + 0, + mockCompletionLabels.length, + ...(additionalCompletions ?? []).map(c => c.label), + ); + return ( +