<commit_before>import memoizeOne from 'memoize-one';

import { Field, FieldType, getParser, LinkModel, LogRowModel } from '@grafana/data';

import { MAX_CHARACTERS } from './LogRowMessage';

const memoizedGetParser = memoizeOne(getParser);

type FieldDef = {
  key: string;
  value: string;
  links?: Array<LinkModel<Field>>;
  fieldIndex?: number;
};

/**
 * Returns all fields for log row which consists of fields we parse from the message itself and any derived fields
 * setup in data source config.
 */
export const getAllFields = memoizeOne(
  (row: LogRowModel, getFieldLinks?: (field: Field, rowIndex: number) => Array<LinkModel<Field>>) => {
    const fields = parseMessage(row.entry);
    const derivedFields = getDerivedFields(row, getFieldLinks);
    const fieldsMap = [...derivedFields, ...fields].reduce((acc, field) => {
      // Strip enclosing quotes for hashing. When values are parsed from log line the quotes are kept, but if same
      // value is in the dataFrame it will be without the quotes. We treat them here as the same value.
      const value = field.value.replace(/(^")|("$)/g, '');
      const fieldHash = `${field.key}=${value}`;
      if (acc[fieldHash]) {
        acc[fieldHash].links = [...(acc[fieldHash].links || []), ...(field.links || [])];
      } else {
        acc[fieldHash] = field;
      }
      return acc;
    }, {} as { [key: string]: FieldDef });

    const allFields = Object.values(fieldsMap);
    allFields.sort(sortFieldsLinkFirst);

    return allFields;
  }
);

const parseMessage = memoizeOne((rowEntry): FieldDef[] => {
  if (rowEntry.length > MAX_CHARACTERS) {
    return [];
  }
  const parser = memoizedGetParser(rowEntry);
  if (!parser) {
    return [];
  }
  // Use parser to highlight detected fields
  const detectedFields = parser.getFields(rowEntry);
  const fields = detectedFields.map((field) => {
    const key = parser.getLabelFromField(field);
    const value = parser.getValueFromField(field);
    return { key, value };
  });

  return fields;
});

const getDerivedFields = memoizeOne(
  (row: LogRowModel, getFieldLinks?: (field: Field, rowIndex: number) => Array<LinkModel<Field>>): FieldDef[] => {
    return row.dataFrame.fields
      .map((field, index) => ({ ...field, index }))
      .filter((field, index) => !shouldRemoveField(field, index, row))
      .map((field) => {
        const links = getFieldLinks ? getFieldLinks(field, row.rowIndex) : [];
        return {
          key: field.name,
          value: field.values.get(row.rowIndex).toString(),
          links: links,
          fieldIndex: field.index,
        };
      });
  }
);<commit_msg>logs: better names and comments (#53770)<commit_after>import memoizeOne from 'memoize-one';

import { Field, FieldType, getParser, LinkModel, LogRowModel } from '@grafana/data';

import { MAX_CHARACTERS } from './LogRowMessage';

const memoizedGetParser = memoizeOne(getParser);

type FieldDef = {
  key: string;
  value: string;
  links?: Array<LinkModel<Field>>;
  fieldIndex?: number;
};

/**
 * Returns all fields for log row which consists of fields we parse from the message itself and additional fields
 * found in the dataframe (they may contain links).
 */
export const getAllFields = memoizeOne(
  (row: LogRowModel, getFieldLinks?: (field: Field, rowIndex: number) => Array<LinkModel<Field>>) => {
    const logMessageFields = parseMessage(row.entry);
    const dataframeFields = getDataframeFields(row, getFieldLinks);
    const fieldsMap = [...dataframeFields, ...logMessageFields].reduce((acc, field) => {
      // Strip enclosing quotes for hashing. When values are parsed from log line the quotes are kept, but if same
      // value is in the dataFrame it will be without the quotes. We treat them here as the same value.
      // We need to handle this scenario:
      // - we use derived-fields in Loki
      // - we name the derived field the same as the parsed-field-name
      // - the same field will appear twice
      //   - in the fields coming from `logMessageFields`
      //   - in the fields coming from `dataframeFields`
      // - but, in the fields coming from `logMessageFields`, there might be doublequotes around the value
      // - we want to "merge" data from both sources, so we remove quotes from the beginning and end
      const value = field.value.replace(/(^")|("$)/g, '');
      const fieldHash = `${field.key}=${value}`;
      if (acc[fieldHash]) {
        acc[fieldHash].links = [...(acc[fieldHash].links || []), ...(field.links || [])];
      } else {
        acc[fieldHash] = field;
      }
      return acc;
    }, {} as { [key: string]: FieldDef });

    const allFields = Object.values(fieldsMap);
    allFields.sort(sortFieldsLinkFirst);

    return allFields;
  }
);

const parseMessage = memoizeOne((rowEntry): FieldDef[] => {
  if (rowEntry.length > MAX_CHARACTERS) {
    return [];
  }
  const parser = memoizedGetParser(rowEntry);
  if (!parser) {
    return [];
  }
  // Use parser to highlight detected fields
  const detectedFields = parser.getFields(rowEntry);
  const fields = detectedFields.map((field) => {
    const key = parser.getLabelFromField(field);
    const value = parser.getValueFromField(field);
    return { key, value };
  });

  return fields;
});

// creates fields from the dataframe-fields, adding data-links, when field.config.links exists
const getDataframeFields = memoizeOne(
  (row: LogRowModel, getFieldLinks?: (field: Field, rowIndex: number) => Array<LinkModel<Field>>): FieldDef[] => {
    return row.dataFrame.fields
      .map((field, index) => ({ ...field, index }))
      .filter((field, index) => !shouldRemoveField(field, index, row))
      .map((field) => {
        const links = getFieldLinks ? getFieldLinks(field, row.rowIndex) : [];
        return {
          key: field.name,
          value: field.values.get(row.rowIndex).toString(),
          links: links,
          fieldIndex: field.index,
        };
      });
  }
);
{'fraction_non_alphanumeric': 0.1361959074136196, 'fraction_numerical': 0.0008386447500838644, 'mean_word_length': 3.217114568599717, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 37, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6955533', 'n_tokens_mistral': 1747, 'n_tokens_neox': 1736, 'n_words': 521}