RFC: Use ResourceAttributes and ScopeAttributes for log labels as well

src/components/queryBuilder/ColumnsEditor.tsx

@@ -12,6 +12,8 @@ interface ColumnsEditorProps {
   onSelectedColumnsChange: (selectedColumns: SelectedColumn[]) => void;
   disabled?: boolean;
   showAllOption?: boolean;
+  label?: string;
+  tooltip?: string;
 }
 
 function getCustomColumns(columnNames: string[], allColumns: readonly TableColumn[]): Array<SelectableValue<string>> {
@@ -24,15 +26,17 @@ function getCustomColumns(columnNames: string[], allColumns: readonly TableColum
 const allColumnName = '*';
 
 export const ColumnsEditor = (props: ColumnsEditorProps) => {
-  const { allColumns, selectedColumns, onSelectedColumnsChange, disabled, showAllOption } = props;
+  const {
+    allColumns, selectedColumns, onSelectedColumnsChange, disabled, showAllOption,
+    label = labels.components.ColumnsEditor.label, tooltip = labels.components.ColumnsEditor.tooltip
+  } = props;
   const [customColumns, setCustomColumns] = useState<Array<SelectableValue<string>>>([]);
   const [isOpen, setIsOpen] = useState(false);
   const allColumnNames = allColumns.map(c => ({ label: c.label || c.name, value: c.name }));
   if (showAllOption) {
     allColumnNames.push({ label: allColumnName, value: allColumnName });
   }
   const selectedColumnNames = (selectedColumns || []).map(c => ({ label: c.alias || c.name, value: c.name }));
-  const { label, tooltip } = labels.components.ColumnsEditor;
 
   const options = [...allColumnNames, ...customColumns];
 

src/components/queryBuilder/views/LogsQueryBuilder.tsx

@@ -32,7 +32,7 @@ interface LogsQueryBuilderState {
   timeColumn?: SelectedColumn;
   logLevelColumn?: SelectedColumn;
   messageColumn?: SelectedColumn;
-  labelsColumn?: SelectedColumn;
+  labelsColumns: SelectedColumn[];
   // liveView: boolean;
   orderBy: OrderBy[];
   limit: number;
@@ -78,8 +78,9 @@ export const LogsQueryBuilder = (props: LogsQueryBuilderProps) => {
     if (next.messageColumn) {
       nextColumns.push(next.messageColumn);
     }
-    if (next.labelsColumn) {
-      nextColumns.push(next.labelsColumn);
+
+    for (const c of next.labelsColumns) {
+      nextColumns.push({ ...c, mapForLabels: true })
     }
 
     builderOptionsDispatch(setOptions({
@@ -161,16 +162,13 @@ export const LogsQueryBuilder = (props: LogsQueryBuilderProps) => {
           label={labels.logMessageColumn.label}
           tooltip={labels.logMessageColumn.tooltip}
         />
-        <ColumnSelect
+        <ColumnsEditor
           disabled={builderState.otelEnabled}
           allColumns={allColumns}
-          selectedColumn={builderState.labelsColumn}
-          invalid={!builderState.labelsColumn}
-          onColumnChange={onOptionChange('labelsColumn')}
-          columnHint={ColumnHint.LogLabels}
+          selectedColumns={builderState.labelsColumns}
+          onSelectedColumnsChange={onOptionChange('labelsColumns')}
           label={labels.logLabelsColumn.label}
           tooltip={labels.logLabelsColumn.tooltip}
-          inline
         />
         {/* <Switch
           value={builderState.liveView}

src/components/queryBuilder/views/logsQueryBuilderHooks.ts

@@ -23,7 +23,8 @@ export const useLogDefaultsOnMount = (datasource: Datasource, isNewQuery: boolea
     const nextColumns: SelectedColumn[] = [];
     const includedColumns = new Set<string>();
     for (let [hint, colName] of defaultColumns) {
-      nextColumns.push({ name: colName, hint });
+      const mapForLabels = (hint === ColumnHint.LogAttributes || hint === ColumnHint.LogResourceAttributes || hint === ColumnHint.LogScopeAttributes)
+      nextColumns.push({ name: colName, hint, mapForLabels });
       includedColumns.add(colName);
     }
 
@@ -77,7 +78,8 @@ export const useOtelColumns = (datasource: Datasource, otelEnabled: boolean, ote
     const columns: SelectedColumn[] = [];
     const includedColumns = new Set<string>();
     logColumnMap.forEach((name, hint) => {
-      columns.push({ name, hint });
+      const mapForLabels = (hint === ColumnHint.LogAttributes || hint === ColumnHint.LogResourceAttributes || hint === ColumnHint.LogScopeAttributes)
+      columns.push({ name, hint, mapForLabels });
       includedColumns.add(name);
     });
 

src/data/CHDatasource.ts

@@ -159,6 +159,7 @@ export class Datasource
       for (level in LOG_LEVEL_TO_IN_CLAUSE) {
         aggregates.push({ aggregateType: AggregateType.Sum, column: `multiSearchAny(${llf}, [${LOG_LEVEL_TO_IN_CLAUSE[level]}])`, alias: level });
       }
+      columns.push({name: `count(*) - (${Object.keys(LOG_LEVEL_TO_IN_CLAUSE).join('+')})`, alias: 'unknown'})
     } else {
       // Count all logs if level column isn't selected
       aggregates.push({
@@ -292,9 +293,21 @@ export class Datasource
     const lookupByAlias = query.builderOptions.columns?.find(c => c.alias === columnName); // Check all aliases first,
     const lookupByName = query.builderOptions.columns?.find(c => c.name === columnName);   // then try matching column name
     const lookupByLogsAlias = logAliasToColumnHints.has(columnName) ? getColumnByHint(query.builderOptions, logAliasToColumnHints.get(columnName)!) : undefined;
-    const lookupByLogLabels = dataFrameHasLogLabelWithName(actionFrame, columnName) && getColumnByHint(query.builderOptions, ColumnHint.LogLabels);
+
+    let lookupByLogLabels: SelectedColumn | undefined = undefined;
+    let mapKey: string | undefined = undefined;
+    const labelColumn = dataFrameHasLogLabelWithName(actionFrame, columnName);
+    if (labelColumn) {
+      if (labelColumn.type === 'hint') {
+        lookupByLogLabels = getColumnByHint(query.builderOptions, labelColumn.hint)
+      } else {
+        lookupByLogLabels = query.builderOptions.columns?.find(c => c.name === labelColumn.name)
+      }
+      mapKey = labelColumn.mapKey
+    }
+
     const column = lookupByAlias || lookupByName || lookupByLogsAlias || lookupByLogLabels;
-    
+
     let nextFilters: Filter[] = (query.builderOptions.filters?.slice() || []);
     if (action.type === 'ADD_FILTER') {
       // we need to remove *any other EQ or NE* for the same field,
@@ -307,16 +320,16 @@ export class Datasource
         ) &&
         !(
           f.type.toLowerCase().startsWith('map') &&
-          (column && lookupByLogLabels && f.mapKey === columnName) &&
+          (column && lookupByLogLabels && f.key === lookupByLogLabels.name && f.mapKey === mapKey) &&
           (f.operator === FilterOperator.IsAnything || f.operator === FilterOperator.Equals || f.operator === FilterOperator.NotEquals)
         )
       );
 
       nextFilters.push({
         condition: 'AND',
-        key: (column && column.hint) ? '' : columnName,
+        key: lookupByLogLabels ? lookupByLogLabels.name : columnName,
         hint: (column && column.hint) ? column.hint : undefined,
-        mapKey: lookupByLogLabels ? columnName : undefined,
+        mapKey: lookupByLogLabels ? mapKey : undefined,
         type: lookupByLogLabels ? 'Map(String, String)' : 'string',
         filterType: 'custom',
         operator: FilterOperator.Equals,
@@ -340,17 +353,17 @@ export class Datasource
           ) ||
           (
             f.type.toLowerCase().startsWith('map') &&
-            (column && lookupByLogLabels && f.mapKey === columnName) &&
+            (column && lookupByLogLabels && f.key === lookupByLogLabels.name && f.mapKey === mapKey) &&
             (f.operator === FilterOperator.IsAnything || f.operator === FilterOperator.Equals)
           )
         )
       );
 
       nextFilters.push({
         condition: 'AND',
-        key: (column && column.hint) ? '' : columnName,
+        key:  lookupByLogLabels ? lookupByLogLabels.name : columnName,
         hint: (column && column.hint) ? column.hint : undefined,
-        mapKey: lookupByLogLabels ? columnName : undefined,
+        mapKey: lookupByLogLabels ? mapKey : undefined,
         type: lookupByLogLabels ? 'Map(String, String)' : 'string',
         filterType: 'custom',
         operator: FilterOperator.NotEquals,

src/data/sqlGenerator.ts

@@ -222,7 +222,7 @@ const generateTraceIdQuery = (options: QueryBuilderOptions): string => {
  */
 const generateLogsQuery = (_options: QueryBuilderOptions): string => {
   // Copy columns so column aliases can be safely mutated
-  const options = { ..._options, columns: _options.columns?.map(c => ({ ...c })) };
+  const options = { ..._options, columns: _options.columns?.map(c => ({ ...c })) ?? [] };
   const { database, table } = options;
 
   const queryParts: string[] = [];
@@ -250,10 +250,20 @@ const generateLogsQuery = (_options: QueryBuilderOptions): string => {
     selectParts.push(getColumnIdentifier(logLevel));
   }
 
-  const logLabels = getColumnByHint(options, ColumnHint.LogLabels);
-  if (logLabels !== undefined) {
-    logLabels.alias = logColumnHintsToAlias.get(ColumnHint.LogLabels);
-    selectParts.push(getColumnIdentifier(logLabels));
+  const logLabelsCols = options.columns.filter(c => c.mapForLabels);
+  if (0 < logLabelsCols.length) {
+    const mapExprs = [];
+    for (const col of logLabelsCols) {
+      const prefix = new Map<ColumnHint | undefined, string>([
+        [ColumnHint.LogAttributes, 'attr'],
+        [ColumnHint.LogResourceAttributes, 'res'],
+        [ColumnHint.LogScopeAttributes, 'span']
+      ]).get(col.hint) ?? col.name;
+      const c = escapeIdentifier(col.name);
+      const mapExpr = `mapApply((k, v) -> ('${prefix}.' || k,  v), ${c})`
+      mapExprs.push(mapExpr);
+    }
+    selectParts.push(`mapConcat(${mapExprs.join(',')}) as ${escapeIdentifier(LABELS_ALIAS)}`);
   }
 
   const traceId = getColumnByHint(options, ColumnHint.TraceId);
@@ -797,9 +807,9 @@ const logAliasToColumnHintsEntries: ReadonlyArray<[string, ColumnHint]> = [
   ['timestamp', ColumnHint.Time],
   ['body', ColumnHint.LogMessage],
   ['level', ColumnHint.LogLevel],
-  ['labels', ColumnHint.LogLabels],
   ['traceID', ColumnHint.TraceId],
 ];
+export const LABELS_ALIAS = 'labels';
 export const logAliasToColumnHints: Map<string, ColumnHint> = new Map(logAliasToColumnHintsEntries);
 export const logColumnHintsToAlias: Map<ColumnHint, string> = new Map(logAliasToColumnHintsEntries.map(e => [e[1], e[0]]));
 

src/data/utils.test.ts

@@ -3,7 +3,7 @@ import { columnLabelToPlaceholder, dataFrameHasLogLabelWithName, isBuilderOption
 import { newMockDatasource } from "__mocks__/datasource";
 import { CoreApp, DataFrame, DataQueryRequest, DataQueryResponse, Field, FieldType } from "@grafana/data";
 import { CHBuilderQuery, CHQuery, EditorType } from "types/sql";
-import { logColumnHintsToAlias } from "./sqlGenerator";
+import { LABELS_ALIAS } from "./sqlGenerator";
 
 describe('isBuilderOptionsRunnable', () => {
   it('should return false for empty builder options', () => {
@@ -209,7 +209,7 @@ describe('transformQueryResponseWithTraceAndLogLinks', () => {
 
 
 describe('dataFrameHasLogLabelWithName', () => {
-  const logLabelsFieldName = logColumnHintsToAlias.get(ColumnHint.LogLabels);
+  const logLabelsFieldName = LABELS_ALIAS;
 
   it('should return false for undefined dataframe', () => {
     expect(dataFrameHasLogLabelWithName(undefined, 'testLabel')).toBe(false);

src/data/utils.ts

@@ -3,7 +3,7 @@ import { ColumnHint, FilterOperator, OrderByDirection, QueryBuilderOptions, Quer
 import { CHBuilderQuery, CHQuery, EditorType } from "types/sql";
 import { Datasource } from "./CHDatasource";
 import { pluginVersion } from "utils/version";
-import { logColumnHintsToAlias } from "./sqlGenerator";
+import { LABELS_ALIAS } from "./sqlGenerator";
 
 /**
  * Returns true if the builder options contain enough information to start showing a query
@@ -281,33 +281,62 @@ export const transformQueryResponseWithTraceAndLogLinks = (datasource: Datasourc
         datasourceUid: traceLogsQuery.datasource?.uid!,
         datasourceName: traceLogsQuery.datasource?.type!,
       }
-    }); 
+    });
   });
 
   return res;
 };
 
 
+type FoundLabelWithColumnHint = {
+  type: 'hint',
+  hint: ColumnHint
+  mapKey: string,
+}
+type FoundLabelWithColumnName = {
+  type: 'name',
+  name: string,
+  mapKey: string,
+}
+
 /**
- * Returns true if the dataframe contains a log label that matches the provided name.
- * 
+ * Returns the source labels column if the dataframe contains a log label that matches the provided name.
+ *
  * This function exists for the logs panel, when clicking "filter for value" on a single log row.
  * A dataframe will be provided for that single row, and we need to check the labels object to see if it
- * contains a field with that name. If it does then we can create a filter using the labels column hint.
+ * contains a field with that name. If it does then we can create a filter using the labels column.
  */
-export const dataFrameHasLogLabelWithName = (frame: DataFrame | undefined, name: string): boolean => {
+export const dataFrameHasLogLabelWithName = (frame: DataFrame | undefined, name: string): FoundLabelWithColumnHint | FoundLabelWithColumnName | undefined => {
   if (!frame || !frame.fields || frame.fields.length === 0) {
-    return false;
+    return undefined;
   }
 
-  const logLabelsFieldName = logColumnHintsToAlias.get(ColumnHint.LogLabels);
-  const field = frame.fields.find(f => f.name === logLabelsFieldName);
+  const field = frame.fields.find(f => f.name === LABELS_ALIAS);
   if (!field || !field.values || field.values.length < 1 || !field.values.get(0)) {
-    return false;
+    return undefined;
   }
 
   const labels = (field.values.get(0) || {}) as object;
   const labelKeys = Object.keys(labels);
 
-  return labelKeys.includes(name);
+  if (!labelKeys.includes(name)) {
+    return undefined;
+  }
+
+  const {prefix, mapKey} = /^(?<prefix>.+?)\.(?<mapKey>.+)$/.exec(name)?.groups ?? {};
+  if (!prefix || !mapKey) {
+    return undefined
+  }
+
+  const hint = new Map<string|undefined, ColumnHint>(Object.entries({
+    'attr': ColumnHint.LogAttributes,
+    'res': ColumnHint.LogResourceAttributes,
+    'scope': ColumnHint.LogScopeAttributes
+  })).get(prefix);
+
+  if (hint) {
+    return {type: 'hint', hint, mapKey}
+  } else {
+    return {type: 'name', name: prefix, mapKey}
+  }
 }

src/labels.ts

@@ -452,7 +452,10 @@ export default {
 
       [ColumnHint.LogLevel]: 'Level',
       [ColumnHint.LogMessage]: 'Message',
-      [ColumnHint.LogLabels]: 'Labels',
+
+      [ColumnHint.LogAttributes]: 'Log Attributes',
+      [ColumnHint.LogResourceAttributes]: 'Resource Attributes',
+      [ColumnHint.LogScopeAttributes]: 'Scope Attributes',
 
       [ColumnHint.TraceId]: 'Trace ID',
       [ColumnHint.TraceSpanId]: 'Span ID',

src/otel.ts

@@ -26,7 +26,9 @@ const otel129: OtelVersion = {
     [ColumnHint.Time, 'Timestamp'],
     [ColumnHint.LogMessage, 'Body'],
     [ColumnHint.LogLevel, 'SeverityText'],
-    [ColumnHint.LogLabels, 'LogAttributes'],
+    [ColumnHint.LogAttributes, 'LogAttributes'],
+    [ColumnHint.LogResourceAttributes, 'ResourceAttributes'],
+    [ColumnHint.LogScopeAttributes, 'ScopeAttributes'],
     [ColumnHint.TraceId, 'TraceId'],
   ]),
   logLevels: [

src/types/queryBuilder.ts

@@ -122,7 +122,10 @@ export enum ColumnHint {
 
   LogLevel = 'log_level',
   LogMessage = 'log_message',
-  LogLabels = 'log_labels',
+
+  LogAttributes = 'log_attributes',
+  LogResourceAttributes = 'log_resource_attributes',
+  LogScopeAttributes = 'log_scope_attributes',
 
   TraceId = 'trace_id',
   TraceSpanId = 'trace_span_id',
@@ -155,6 +158,7 @@ export interface SelectedColumn {
   alias?: string;
   custom?: boolean;
   hint?: ColumnHint;
+  mapForLabels?: boolean
 }
 
 export enum OrderByDirection {