<commit_before>/* This Source Code Form is subject to the terms of the Mozilla Public
 * License, v. 2.0. If a copy of the MPL was not distributed with this
 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */

// @flow
import {
  getEmptyProfile,
  getEmptyThread,
  getEmptyJsTracerTable,
  resourceTypes,
  getEmptyJsAllocationsTable,
  getEmptyUnbalancedNativeAllocationsTable,
  getEmptyBalancedNativeAllocationsTable,
} from '../../../profile-logic/data-structures';
import { mergeProfilesForDiffing } from '../../../profile-logic/merge-compare';
import { stateFromLocation } from '../../../app-logic/url-handling';
import { UniqueStringArray } from '../../../utils/unique-string-array';
import { ensureExists } from '../../../utils/flow';
import {
  INTERVAL,
  INSTANT,
  INTERVAL_START,
  INTERVAL_END,
} from 'firefox-profiler/app-logic/constants';

import type {
  Profile,
  Thread,
  ThreadIndex,
  IndexIntoCategoryList,
  CategoryList,
  JsTracerTable,
  Counter,
  TabID,
  MarkerPayload,
  NetworkPayload,
  NavigationMarkerPayload,
  IPCMarkerPayload,
  UserTimingMarkerPayload,
  Milliseconds,
  MarkerPhase,
  ThreadCPUDeltaUnit,
  LineNumber,
} from 'firefox-profiler/types';
import {
  deriveMarkersFromRawMarkerTable,
  IPCMarkerCorrelations,
} from '../../../profile-logic/marker-data';
import { getTimeRangeForThread } from '../../../profile-logic/profile-data';
import { markerSchemaForTests } from './marker-schema';

// Array<[MarkerName, Milliseconds, Data]>
type MarkerName = string;
type MarkerTime = Milliseconds;
type MockPayload = {| startTime: Milliseconds, endTime: Milliseconds |};

// These markers can create an Instant or a complete Interval marker, depending
// on if an end time is passed in. The definition uses a union, becaus as far
// as I can tell, Flow doesn't support multiple arity tuples.
export type TestDefinedMarkers = Array<
  // Instant marker:
  | [MarkerName, MarkerTime]
  // No payload:
  | [
      MarkerName,
      MarkerTime, // start time
      MarkerTime | null // end time
    ]
  | [
      MarkerName,
      MarkerTime, // start time
      MarkerTime | null, // end time
      MarkerPayload | MockPayload
    ]
>;

// This type is used when needing to create a specific RawMarkerTable.
export type TestDefinedRawMarker = {|
  +name?: string,
  +startTime: Milliseconds | null,
  +endTime: Milliseconds | null,
  +phase: MarkerPhase,
  +category?: IndexIntoCategoryList,
  +data?: MarkerPayload,
|};

export type TestDefinedJsTracerEvent = [
  // Event name:
  string,
  // Start time:
  Milliseconds,
  // End time:
  Milliseconds
];

export function addRawMarkersToThread(
  thread: Thread,
  markers: TestDefinedRawMarker[]
) {
  const stringTable = thread.stringTable;
  const markersTable = thread.markers;

  for (const { name, startTime, endTime, phase, category, data } of markers) {
    markersTable.name.push(
      stringTable.indexForString(name || 'TestDefinedMarker')
    );
    markersTable.phase.push(phase);
    markersTable.startTime.push(startTime);
    markersTable.endTime.push(endTime);
    markersTable.data.push(data || null);
    markersTable.category.push(category || 0);
    markersTable.length++;
  }
}

export function addMarkersToThreadWithCorrespondingSamples(
  thread: Thread,
  markers: TestDefinedMarkers
) {
  const stringTable = thread.stringTable;
  const markersTable = thread.markers;
  const allTimes = new Set();

  markers.forEach((tuple) => {
    const name = tuple[0];
    const startTime = tuple[1];
    // Flow doesn't support variadic tuple types.
    const maybeEndTime = (tuple: any)[2] || null;
    const payload: MarkerPayload = (tuple: any)[3] || null;

    markersTable.name.push(stringTable.indexForString(name));
    if (maybeEndTime === null) {
      markersTable.phase.push(INSTANT);
      markersTable.startTime.push(startTime);
      markersTable.endTime.push(null);
    } else {
      markersTable.phase.push(INTERVAL);
      markersTable.startTime.push(startTime);
      markersTable.endTime.push(maybeEndTime);
      allTimes.add(maybeEndTime);
    }
    allTimes.add(startTime);
    markersTable.data.push(payload);
    markersTable.category.push(0);
    markersTable.length++;
  });

  // When the profile contains at least 1 sample, we use only the samples to
  // control the initial range. Because of that we need to add samples so that
  // the range includes these markers. Note that when a thread has no sample,
  // then the markers are used to compute the initial range.
  const { samples } = thread;
  if (samples.length) {
    const firstMarkerTime = Math.min(...allTimes);
    const lastMarkerTime = Math.max(...allTimes);

    // The first marker time should be added if there's no sample before this time.
    const shouldAddFirstMarkerTime = samples.time[0] > firstMarkerTime;

    // The last marker time should be added if there's no sample after this time,
    // but only if it's different than the other time.
    const shouldAddLastMarkerTime =
      samples.time[samples.length - 1] < lastMarkerTime &&
      firstMarkerTime !== lastMarkerTime;

    if (shouldAddFirstMarkerTime) {
      samples.time.unshift(firstMarkerTime);
      samples.stack.unshift(null);
      if (samples.responsiveness) {
        samples.responsiveness.unshift(null);
      }
      if (samples.eventDelay) {
        samples.eventDelay.unshift(null);
      }
      if (samples.weight) {
        samples.weight.unshift(samples.weight[0]);
      }
      samples.length++;
    }

    if (shouldAddLastMarkerTime) {
      samples.time.push(lastMarkerTime);
      samples.stack.push(null);
      if (samples.responsiveness) {
        samples.responsiveness.push(null);
      }
      if (samples.eventDelay) {
        samples.eventDelay.push(null);
      }
      if (samples.weight) {
        samples.weight.push(samples.weight[0]);
      }
      samples.length++;
    }
  }
}

export function getThreadWithMarkers(markers: TestDefinedMarkers) {
  const thread = getEmptyThread();
  addMarkersToThreadWithCorrespondingSamples(thread, markers);
  return thread;
}

export function getThreadWithRawMarkers(markers: TestDefinedRawMarker[]) {
  const thread = getEmptyThread();
  addRawMarkersToThread(thread, markers);
  return thread;
}

/**
 * This can be a little annoying to derive with all of the dependencies,
 * so provide an easy interface to do so here.
 */
export function getTestFriendlyDerivedMarkerInfo(thread: Thread) {
  return deriveMarkersFromRawMarkerTable(
    thread.markers,
    thread.stringTable,
    thread.tid || 0,
    getTimeRangeForThread(thread, 1),
    new IPCMarkerCorrelations()
  );
}

/**
 * A utility to make TestDefinedRawMarker
 */
export function makeStartMarker(
  name: string,
  startTime: Milliseconds
): TestDefinedRawMarker {
  return {
    name,
    startTime,
    endTime: null,
    phase: INTERVAL_START,
  };
}

/**
 * A utility to make TestDefinedRawMarker
 */
export function makeEndMarker(
  name: string,
  endTime: Milliseconds
): TestDefinedRawMarker {
  return {
    name,
    startTime: null,
    endTime,
    phase: INTERVAL_END,
  };
}

/**
 * A utility to make TestDefinedRawMarker
 */
export function makeInstantMarker(
  name: string,
  startTime: Milliseconds
): TestDefinedRawMarker {
  return {
    name,
    startTime,
    endTime: null,
    phase: INSTANT,
  };
}

/**
 * A utility to make TestDefinedRawMarker
 */
export function makeIntervalMarker(
  name: string,
  startTime: Milliseconds,
  endTime: Milliseconds
): TestDefinedRawMarker {
  return {
    name,
    startTime,
    endTime,
    phase: INTERVAL,
  };
}

/**
 * A utility to make TestDefinedRawMarker
 */
export function makeCompositorScreenshot(
  startTime: Milliseconds
): TestDefinedRawMarker {
  return {
    ...makeInstantMarker('CompositorScreenshot', startTime),
    data: {
      type: 'CompositorScreenshot',
      url: 0,
      windowID: '',
      windowWidth: 100,
      windowHeight: 100,
    },
  };
}

export function getUserTiming(
  name: string,
  startTime: number,
  duration: number
) {
  const endTime = startTime + duration;
  return [
    'UserTiming',
    startTime,
    endTime,
    ({
      type: 'UserTiming',
      name,
      entryType: 'measure',
    }: UserTimingMarkerPayload),
  ];
}

export function getProfileWithMarkers(
  ...markersPerThread: TestDefinedMarkers[]
): Profile {
  const profile = getEmptyProfile();
  // Provide a useful marker schema, rather than an empty one.
  profile.meta.markerSchema = markerSchemaForTests;

  if (markersPerThread.length === 0) {
    throw new Error(
      'getProfileWithMarkers expected to get at least one list of markers.'
    );
  }
  profile.threads = markersPerThread.map((testDefinedMarkers) =>
    getThreadWithMarkers(testDefinedMarkers)
  );
  return profile;
}

/**
 * This profile is useful for marker table tests. The markers were taken from
 * real-world values.
 */
export function getMarkerTableProfile() {
  return getProfileWithMarkers(
    [
      [
        'UserTiming',
        12.5,
        12.5,
        {
          type: 'UserTiming',
          name: 'foobar',
          entryType: 'mark',
        },
      ],
      [
        'NotifyDidPaint',
        14.5,
        null,
        {
          type: 'tracing',
          category: 'Paint',
        },
      ],
      [
        'setTimeout',
        165.87091900000001,
        165.871503,
        {
          type: 'Text',
          name: '5.5',
        },
      ],
      [
        'IPC',
        120,
        120,
        {
          type: 'IPC',
          startTime: 120,
          endTime: 120,
          otherPid: 2222,
          messageType: 'PContent::Msg_PreferenceUpdate',
          messageSeqno: 1,
          side: 'parent',
          direction: 'sending',
          phase: 'endpoint',
          sync: false,
          niceDirection: 'sending to 2222',
        },
      ],
      [
        'LogMessages',
        170,
        null,
        {
          type: 'Log',
          name: 'nsJARChannel::nsJARChannel [this=0x87f1ec80]\n',
          module: 'nsJarProtocol',
        },
      ],
      [
        'Lorem ipsum dolor sit amet, consectetur adipiscing elit. Sed eget magna sed magna vehicula congue id id nulla. Ut convallis, neque consequat aliquam egestas, dui urna interdum quam, id semper magna erat et nisi. Vivamus molestie quis ligula eget aliquam. Sed facilisis, turpis sed facilisis posuere, risus odio convallis velit, vitae vehicula justo risus at ipsum. Proin non porttitor neque. Vivamus fringilla ex nec iaculis cursus. Vestibulum suscipit mauris sem, vitae gravida ipsum fermentum id. Quisque pulvinar blandit ullamcorper. Donec id justo at metus scelerisque pulvinar. Proin suscipit suscipit nisi, quis tempus ipsum vulputate quis. Pellentesque sodales rutrum eros, eget pulvinar ante condimentum a. Donec accumsan, ante ut facilisis cursus, nibh quam congue eros, vitae placerat tortor magna vel lacus. Etiam odio diam, venenatis eu sollicitudin non, ultrices ut urna. Aliquam vehicula diam eu eros eleifend, ac vulputate purus faucibus.',
        165.87091900000001,
        165.871503,
        {
          type: 'Text',
          name: '5.5',
        },
      ],
      [
        'FileIO',
        174,
        175,
        {
          type: 'FileIO',
          source: 'PoisonIOInterposer',
          filename: '/foo/bar',
          operation: 'create/open',
        },
      ],
    ].sort((a, b) => a[1] - b[1])
  );
}

export function getProfileWithNamedThreads(threadNames: string[]): Profile {
  const profile = getEmptyProfile();
  profile.threads = threadNames.map((name) => getEmptyThread({ name }));
  return profile;
}

/**
 * Create a profile from text representation of samples. Each column in the text provided
 * represents a sample. Each sample is made up of a list of functions.
 * Each column needs to be separated by at least two spaces.
 *
 * Example usage:
 *
 * ```
 * const { profile } = getProfileFromTextSamples(`
 *   A             A             A                A
 *   B.js          B.js          F[cat:Graphics]  F
 *   C.js          C.js          G                G
 *   D             D                              H[cat:GC / CC]
 *   E[cat:Other]  E [cat:Other]
 * `);
 * ```
 *
 * The function names are aligned vertically on the left. This would produce 4 samples
 * with the stacks based off of those functions listed, with A being the root. Single
 * spaces within a column are permitted, surrounding whitespace is trimmed.
 *
 * Functions ending in "js" are marked as JS functions in the funcTable's isJS
 * column. Functions with categories using the notation "functionName[cat:categoryName]"
 * are annotated with that category in the frameTable's category column.
 *
 * The function returns more information as well, that is:
 * * an array mapping the func indices (IndexIntoFuncTable) to their names
 * * an array mapping the func names to their indices
 *
 * Functions ending in "js" are marked as JS functions in the funcTable's isJS
 * column. It's possible to specify a JIT type by specifying in brackets like
 * this: [jit:baseline] or [jit:ion], right after the function name (see below
 * for an example). The default is no JIT.
 *
 * The time of the sample can also be set by making the first row all numbers:
 * ```
 *   const { profile } = getProfileFromTextSamples(`
 *    0  1  5  6
 *    A  A  A  C
 *       B
 *  `);
 * ```
 *
 * The funcNamesDictPerThread array can be useful when using it like this:
 * ```
 * const {
 *   profile,
 *   funcNamesDictPerThread: [{ A, B, Cjs, D }],
 * } = getProfileFromTextSamples(`
 *    A             A
 *    B             B
 *    Cjs[jit:ion]  Cjs[jit:baseline]
 *    D[cat:DOM]    D[cat:DOM]
 *    E             F
 *  `);
 * ```
 * Now the variables named A B Cjs D directly refer to the func indices and can
 * be used in tests.
 */
export function getProfileFromTextSamples(...allTextSamples: string[]): {
  profile: Profile,
  funcNamesPerThread: Array<string[]>,
  funcNamesDictPerThread: Array<{ [funcName: string]: number }>,
} {
  const profile = getEmptyProfile();
  // Provide a useful marker schema, rather than an empty one.
  profile.meta.markerSchema = markerSchemaForTests;
  const categories = ensureExists(
    profile.meta.categories,
    'Expected to find categories.'
  );

  const funcNamesPerThread = [];
  const funcNamesDictPerThread = [];

  profile.threads = allTextSamples.map((textSamples) => {
    // Process the text.
    const textOnlyStacks = _parseTextSamples(textSamples);

    // See if the first row contains only numbers, if so this is the time of the sample.
    let sampleTimes = null;

    // Check if the first row is made by base 10 integers. 0x200 and other will parse
    // as numbers, but they can be used as valid function names.
    const isFirstRowMadeOfNumbers = textOnlyStacks.every((stack) =>
      /^\d+$/.test(stack[0])
    );
    if (isFirstRowMadeOfNumbers) {
      sampleTimes = textOnlyStacks.map((stack) => parseInt(stack[0]));
      for (const stack of textOnlyStacks) {
        // Remove the number.
        stack.shift();
      }
    }

    // Flatten the textOnlyStacks into into a list of function names.
    const funcNamesSet = new Set();
    const removeModifiers = /\[.*/;
    for (let i = 0; i < textOnlyStacks.length; i++) {
      const textOnlyStack = textOnlyStacks[i];
      for (let j = 0; j < textOnlyStack.length; j++) {
        funcNamesSet.add(textOnlyStack[j].replace(removeModifiers, ''));
      }
    }
    const funcNames = [...funcNamesSet];

    const funcNamesDict = funcNames.reduce((result, item, index) => {
      result[item] = index;
      return result;
    }, {});

    funcNamesPerThread.push(funcNames);
    funcNamesDictPerThread.push(funcNamesDict);

    // Turn this into a real thread.
    return _buildThreadFromTextOnlyStacks(
      textOnlyStacks,
      funcNames,
      categories,
      sampleTimes
    );
  });

  return { profile, funcNamesPerThread, funcNamesDictPerThread };
}

function _getAllMatchRanges(regex, str): Array<{ start: number, end: number }> {
  const ranges = [];

  let match;
  do {
    match = regex.exec(str);
    if (match) {
      ranges.push({ start: match.index, end: match.index + match[0].length });
    }
  } while (match);

  return ranges;
}

function _getColumnPositions(line): number[] {
  const lineWithoutIndent = line.trimLeft();
  const indent = line.length - lineWithoutIndent.length;
  const trimmedLine = line.trim();

  // Find the start and end positions of all consecutive runs of two spaces or more.
  const columnSeparatorRanges = _getAllMatchRanges(/ {2,}/g, trimmedLine);
  return [indent, ...columnSeparatorRanges.map((range) => range.end + indent)];
}

/**
 * Turn a text blob into a list of stacks.
 *
 * e.g:
 * const text = `
 *   A  A
 *   B  B
 *   C  C
 *      D
 *      E
 * `
 *
 * Returns:
 * [
 *   ['A', 'B', 'C'],
 *   ['A', 'B', 'C', 'D', E'],
 * ]
 */
function _parseTextSamples(textSamples: string): Array<string[]> {
  const lines = textSamples.split('\n').filter(
    // Filter out empty lines
    (t) => t
  );
  if (lines.length === 0) {
    throw new Error('Empty text data was sent');
  }

  // Compute the index of where the columns start in the string.
  const columnPositions = _getColumnPositions(lines[0]);

  // Create a table of string cells. Empty cells contain the empty string.
  const rows = lines.map((line) =>
    columnPositions.map((pos, columnIndex) =>
      line.substring(pos, columnPositions[columnIndex + 1]).trim()
    )
  );

  // Transpose the table to go from columns to rows.
  return columnPositions.map((_, columnIndex) => {
    const column = [];
    for (let rowIndex = 0; rowIndex < rows.length; rowIndex++) {
      const value = rows[rowIndex][columnIndex];
      if (!value) {
        break;
      }
      column.push(value);
    }
    return column;
  });
}

const JIT_IMPLEMENTATIONS = ['ion', 'baseline', 'blinterp'];

function _findJitTypeFromFuncName(funcNameWithModifier: string): string | null {
  const findJitTypeResult = /\[jit:([^\]]+)\]/.exec(funcNameWithModifier);
  let jitType = null;
  if (findJitTypeResult) {
    jitType = findJitTypeResult[1];
  }

  if (jitType) {
    if (JIT_IMPLEMENTATIONS.includes(jitType)) {
      return jitType;
    }
    throw new Error(
      `The jitType '${jitType}' is unknown to this tool. Is it a typo or should you update the list of possible values?`
    );
  }

  return null;
}

function _isJsFunctionName(funcName) {
  return funcName.endsWith('js');
}

function _findCategoryFromFuncName(
  funcNameWithModifier: string,
  funcName: string,
  categories: CategoryList
): IndexIntoCategoryList | null {
  const findCategoryResult = /\[cat:([^\]]+)\]/.exec(funcNameWithModifier);
  let categoryName;
  if (findCategoryResult) {
    categoryName = findCategoryResult[1];
  } else if (_isJsFunctionName(funcName)) {
    categoryName = 'JavaScript';
  }

  if (categoryName) {
    const category = categories.findIndex((c) => c.name === categoryName);
    if (category !== -1) {
      return category;
    }
  }

  return null;
}

function _findLibNameFromFuncName(funcNameWithModifier: string): string | null {
  const findLibNameResult = /\[lib:([^\]]+)\]/.exec(funcNameWithModifier);
  if (findLibNameResult) {
    const libName = findLibNameResult[1];
    return libName;
  }

  return null;
}

function _findFileNameFromFuncName(
  funcNameWithModifier: string
): string | null {
  const findFileNameResult = /\[file:([^\]]+)\]/.exec(funcNameWithModifier);
  if (findFileNameResult) {
    const fileName = findFileNameResult[1];
    return fileName;
  }

  return null;
}

function _findLineNumberFromFuncName(
  funcNameWithModifier: string
): LineNumber | null {
  const findLineNumberResult = /\[line:([0-9]+)\]/.exec(funcNameWithModifier);
  if (findLineNumberResult) {
    return +findLineNumberResult[1];
  }

  return null;
}

function _buildThreadFromTextOnlyStacks(
  textOnlyStacks: Array<string[]>,
  funcNames: Array<string>,
  categories: CategoryList,
  sampleTimes: number[] | null
): Thread {
  const thread = getEmptyThread();

  const {
    funcTable,
    stringTable,
    frameTable,
    stackTable,
    samples,
    resourceTable,
    libs,
  } = thread;

  // Create the FuncTable.
  funcNames.forEach((funcName) => {
    funcTable.name.push(stringTable.indexForString(funcName));
    funcTable.fileName.push(null);
    funcTable.relevantForJS.push(funcName.endsWith('js-relevant'));
    funcTable.isJS.push(_isJsFunctionName(funcName));
    funcTable.lineNumber.push(null);
    funcTable.columnNumber.push(null);
    // Ignore resources for now, this way funcNames have really nice string indexes.
    // The resource column will be filled in the loop below.
    funcTable.length++;
  });

  const categoryOther = categories.findIndex((c) => c.name === 'Other');

  // This map caches resource indexes for library names.
  const resourceIndexCache = {};

  // Create the samples, stacks, and frames.
  textOnlyStacks.forEach((column, columnIndex) => {
    let prefix = null;
    column.forEach((funcNameWithModifier) => {
      const funcName = funcNameWithModifier.replace(/\[.*/, '');

      // There is a one-to-one relationship between strings and funcIndexes here, so
      // the indexes can double as both string indexes and func indexes.
      const funcIndex = stringTable.indexForString(funcName);

      // Find the library name from the function name and create an entry if needed.
      const libraryName = _findLibNameFromFuncName(funcNameWithModifier);
      let resourceIndex = -1;
      if (libraryName) {
        resourceIndex = resourceIndexCache[libraryName];
        if (resourceIndex === undefined) {
          libs.push({
            start: 0,
            end: 0,
            offset: 0,
            arch: '',
            name: libraryName,
            path: '/path/to/' + libraryName,
            debugName: libraryName,
            debugPath: '/path/to/' + libraryName,
            breakpadId: 'SOMETHING_FAKE',
          });

          resourceTable.lib.push(libs.length - 1); // The lastly inserted item.
          resourceTable.name.push(stringTable.indexForString(libraryName));
          resourceTable.type.push(resourceTypes.library);
          resourceTable.host.push(undefined);
          resourceIndex = resourceTable.length++;

          resourceIndexCache[libraryName] = resourceIndex;
        }
      }

      funcTable.resource[funcIndex] = resourceIndex;

      // Find the file name from the function name
      const fileName = _findFileNameFromFuncName(funcNameWithModifier);
      if (fileName) {
        funcTable.fileName[funcIndex] = stringTable.indexForString(fileName);
      }

      // Find the wanted jit type from the function name
      const jitType = _findJitTypeFromFuncName(funcNameWithModifier);
      const jitTypeIndex = jitType ? stringTable.indexForString(jitType) : null;
      const category = _findCategoryFromFuncName(
        funcNameWithModifier,
        funcName,
        categories
      );
      const lineNumber = _findLineNumberFromFuncName(funcNameWithModifier);

      // Attempt to find a frame that satisfies the given funcIndex, jit type,
      // category, and line number.
      let frameIndex;
      for (let i = 0; i < frameTable.length; i++) {
        if (
          funcIndex === frameTable.func[i] &&
          jitTypeIndex === frameTable.implementation[i] &&
          category === frameTable.category[i] &&
          lineNumber === frameTable.line[i]
        ) {
          frameIndex = i;
          break;
        }
      }

      if (frameIndex === undefined) {
        frameTable.func.push(funcIndex);
        frameTable.address.push(
          funcName.startsWith('0x') ? parseInt(funcName.substr(2), 16) : -1
        );
        frameTable.inlineDepth.push(0);
        frameTable.category.push(category);
        frameTable.subcategory.push(0);
        frameTable.innerWindowID.push(0);
        frameTable.nativeSymbol.push(null);
        frameTable.implementation.push(jitTypeIndex);
        frameTable.line.push(lineNumber);
        frameTable.column.push(null);
        frameTable.optimizations.push(null);
        frameIndex = frameTable.length++;
      }

      // Attempt to find a stack that satisfies the given frameIndex and prefix.
      let stackIndex;
      for (let i = 0; i < stackTable.length; i++) {
        if (
          stackTable.prefix[i] === prefix &&
          stackTable.frame[i] === frameIndex
        ) {
          stackIndex = i;
          break;
        }
      }

      // If we couldn't find a stack, go ahead and create it.
      if (stackIndex === undefined) {
        const frameCategory = frameTable.category[frameIndex];
        const frameSubcategory = frameTable.subcategory[frameIndex];
        const prefixCategory =<commit_msg>Add plain text profile support for [address:*] and [inl:*].<commit_after>/* This Source Code Form is subject to the terms of the Mozilla Public
 * License, v. 2.0. If a copy of the MPL was not distributed with this
 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */

// @flow
import {
  getEmptyProfile,
  getEmptyThread,
  getEmptyJsTracerTable,
  resourceTypes,
  getEmptyJsAllocationsTable,
  getEmptyUnbalancedNativeAllocationsTable,
  getEmptyBalancedNativeAllocationsTable,
} from '../../../profile-logic/data-structures';
import { mergeProfilesForDiffing } from '../../../profile-logic/merge-compare';
import { stateFromLocation } from '../../../app-logic/url-handling';
import { UniqueStringArray } from '../../../utils/unique-string-array';
import { ensureExists } from '../../../utils/flow';
import {
  INTERVAL,
  INSTANT,
  INTERVAL_START,
  INTERVAL_END,
} from 'firefox-profiler/app-logic/constants';

import type {
  Profile,
  Thread,
  ThreadIndex,
  IndexIntoCategoryList,
  CategoryList,
  JsTracerTable,
  Counter,
  TabID,
  MarkerPayload,
  NetworkPayload,
  NavigationMarkerPayload,
  IPCMarkerPayload,
  UserTimingMarkerPayload,
  Milliseconds,
  MarkerPhase,
  ThreadCPUDeltaUnit,
  LineNumber,
  Address,
} from 'firefox-profiler/types';
import {
  deriveMarkersFromRawMarkerTable,
  IPCMarkerCorrelations,
} from '../../../profile-logic/marker-data';
import { getTimeRangeForThread } from '../../../profile-logic/profile-data';
import { markerSchemaForTests } from './marker-schema';

// Array<[MarkerName, Milliseconds, Data]>
type MarkerName = string;
type MarkerTime = Milliseconds;
type MockPayload = {| startTime: Milliseconds, endTime: Milliseconds |};

// These markers can create an Instant or a complete Interval marker, depending
// on if an end time is passed in. The definition uses a union, becaus as far
// as I can tell, Flow doesn't support multiple arity tuples.
export type TestDefinedMarkers = Array<
  // Instant marker:
  | [MarkerName, MarkerTime]
  // No payload:
  | [
      MarkerName,
      MarkerTime, // start time
      MarkerTime | null // end time
    ]
  | [
      MarkerName,
      MarkerTime, // start time
      MarkerTime | null, // end time
      MarkerPayload | MockPayload
    ]
>;

// This type is used when needing to create a specific RawMarkerTable.
export type TestDefinedRawMarker = {|
  +name?: string,
  +startTime: Milliseconds | null,
  +endTime: Milliseconds | null,
  +phase: MarkerPhase,
  +category?: IndexIntoCategoryList,
  +data?: MarkerPayload,
|};

export type TestDefinedJsTracerEvent = [
  // Event name:
  string,
  // Start time:
  Milliseconds,
  // End time:
  Milliseconds
];

export function addRawMarkersToThread(
  thread: Thread,
  markers: TestDefinedRawMarker[]
) {
  const stringTable = thread.stringTable;
  const markersTable = thread.markers;

  for (const { name, startTime, endTime, phase, category, data } of markers) {
    markersTable.name.push(
      stringTable.indexForString(name || 'TestDefinedMarker')
    );
    markersTable.phase.push(phase);
    markersTable.startTime.push(startTime);
    markersTable.endTime.push(endTime);
    markersTable.data.push(data || null);
    markersTable.category.push(category || 0);
    markersTable.length++;
  }
}

export function addMarkersToThreadWithCorrespondingSamples(
  thread: Thread,
  markers: TestDefinedMarkers
) {
  const stringTable = thread.stringTable;
  const markersTable = thread.markers;
  const allTimes = new Set();

  markers.forEach((tuple) => {
    const name = tuple[0];
    const startTime = tuple[1];
    // Flow doesn't support variadic tuple types.
    const maybeEndTime = (tuple: any)[2] || null;
    const payload: MarkerPayload = (tuple: any)[3] || null;

    markersTable.name.push(stringTable.indexForString(name));
    if (maybeEndTime === null) {
      markersTable.phase.push(INSTANT);
      markersTable.startTime.push(startTime);
      markersTable.endTime.push(null);
    } else {
      markersTable.phase.push(INTERVAL);
      markersTable.startTime.push(startTime);
      markersTable.endTime.push(maybeEndTime);
      allTimes.add(maybeEndTime);
    }
    allTimes.add(startTime);
    markersTable.data.push(payload);
    markersTable.category.push(0);
    markersTable.length++;
  });

  // When the profile contains at least 1 sample, we use only the samples to
  // control the initial range. Because of that we need to add samples so that
  // the range includes these markers. Note that when a thread has no sample,
  // then the markers are used to compute the initial range.
  const { samples } = thread;
  if (samples.length) {
    const firstMarkerTime = Math.min(...allTimes);
    const lastMarkerTime = Math.max(...allTimes);

    // The first marker time should be added if there's no sample before this time.
    const shouldAddFirstMarkerTime = samples.time[0] > firstMarkerTime;

    // The last marker time should be added if there's no sample after this time,
    // but only if it's different than the other time.
    const shouldAddLastMarkerTime =
      samples.time[samples.length - 1] < lastMarkerTime &&
      firstMarkerTime !== lastMarkerTime;

    if (shouldAddFirstMarkerTime) {
      samples.time.unshift(firstMarkerTime);
      samples.stack.unshift(null);
      if (samples.responsiveness) {
        samples.responsiveness.unshift(null);
      }
      if (samples.eventDelay) {
        samples.eventDelay.unshift(null);
      }
      if (samples.weight) {
        samples.weight.unshift(samples.weight[0]);
      }
      samples.length++;
    }

    if (shouldAddLastMarkerTime) {
      samples.time.push(lastMarkerTime);
      samples.stack.push(null);
      if (samples.responsiveness) {
        samples.responsiveness.push(null);
      }
      if (samples.eventDelay) {
        samples.eventDelay.push(null);
      }
      if (samples.weight) {
        samples.weight.push(samples.weight[0]);
      }
      samples.length++;
    }
  }
}

export function getThreadWithMarkers(markers: TestDefinedMarkers) {
  const thread = getEmptyThread();
  addMarkersToThreadWithCorrespondingSamples(thread, markers);
  return thread;
}

export function getThreadWithRawMarkers(markers: TestDefinedRawMarker[]) {
  const thread = getEmptyThread();
  addRawMarkersToThread(thread, markers);
  return thread;
}

/**
 * This can be a little annoying to derive with all of the dependencies,
 * so provide an easy interface to do so here.
 */
export function getTestFriendlyDerivedMarkerInfo(thread: Thread) {
  return deriveMarkersFromRawMarkerTable(
    thread.markers,
    thread.stringTable,
    thread.tid || 0,
    getTimeRangeForThread(thread, 1),
    new IPCMarkerCorrelations()
  );
}

/**
 * A utility to make TestDefinedRawMarker
 */
export function makeStartMarker(
  name: string,
  startTime: Milliseconds
): TestDefinedRawMarker {
  return {
    name,
    startTime,
    endTime: null,
    phase: INTERVAL_START,
  };
}

/**
 * A utility to make TestDefinedRawMarker
 */
export function makeEndMarker(
  name: string,
  endTime: Milliseconds
): TestDefinedRawMarker {
  return {
    name,
    startTime: null,
    endTime,
    phase: INTERVAL_END,
  };
}

/**
 * A utility to make TestDefinedRawMarker
 */
export function makeInstantMarker(
  name: string,
  startTime: Milliseconds
): TestDefinedRawMarker {
  return {
    name,
    startTime,
    endTime: null,
    phase: INSTANT,
  };
}

/**
 * A utility to make TestDefinedRawMarker
 */
export function makeIntervalMarker(
  name: string,
  startTime: Milliseconds,
  endTime: Milliseconds
): TestDefinedRawMarker {
  return {
    name,
    startTime,
    endTime,
    phase: INTERVAL,
  };
}

/**
 * A utility to make TestDefinedRawMarker
 */
export function makeCompositorScreenshot(
  startTime: Milliseconds
): TestDefinedRawMarker {
  return {
    ...makeInstantMarker('CompositorScreenshot', startTime),
    data: {
      type: 'CompositorScreenshot',
      url: 0,
      windowID: '',
      windowWidth: 100,
      windowHeight: 100,
    },
  };
}

export function getUserTiming(
  name: string,
  startTime: number,
  duration: number
) {
  const endTime = startTime + duration;
  return [
    'UserTiming',
    startTime,
    endTime,
    ({
      type: 'UserTiming',
      name,
      entryType: 'measure',
    }: UserTimingMarkerPayload),
  ];
}

export function getProfileWithMarkers(
  ...markersPerThread: TestDefinedMarkers[]
): Profile {
  const profile = getEmptyProfile();
  // Provide a useful marker schema, rather than an empty one.
  profile.meta.markerSchema = markerSchemaForTests;

  if (markersPerThread.length === 0) {
    throw new Error(
      'getProfileWithMarkers expected to get at least one list of markers.'
    );
  }
  profile.threads = markersPerThread.map((testDefinedMarkers) =>
    getThreadWithMarkers(testDefinedMarkers)
  );
  return profile;
}

/**
 * This profile is useful for marker table tests. The markers were taken from
 * real-world values.
 */
export function getMarkerTableProfile() {
  return getProfileWithMarkers(
    [
      [
        'UserTiming',
        12.5,
        12.5,
        {
          type: 'UserTiming',
          name: 'foobar',
          entryType: 'mark',
        },
      ],
      [
        'NotifyDidPaint',
        14.5,
        null,
        {
          type: 'tracing',
          category: 'Paint',
        },
      ],
      [
        'setTimeout',
        165.87091900000001,
        165.871503,
        {
          type: 'Text',
          name: '5.5',
        },
      ],
      [
        'IPC',
        120,
        120,
        {
          type: 'IPC',
          startTime: 120,
          endTime: 120,
          otherPid: 2222,
          messageType: 'PContent::Msg_PreferenceUpdate',
          messageSeqno: 1,
          side: 'parent',
          direction: 'sending',
          phase: 'endpoint',
          sync: false,
          niceDirection: 'sending to 2222',
        },
      ],
      [
        'LogMessages',
        170,
        null,
        {
          type: 'Log',
          name: 'nsJARChannel::nsJARChannel [this=0x87f1ec80]\n',
          module: 'nsJarProtocol',
        },
      ],
      [
        'Lorem ipsum dolor sit amet, consectetur adipiscing elit. Sed eget magna sed magna vehicula congue id id nulla. Ut convallis, neque consequat aliquam egestas, dui urna interdum quam, id semper magna erat et nisi. Vivamus molestie quis ligula eget aliquam. Sed facilisis, turpis sed facilisis posuere, risus odio convallis velit, vitae vehicula justo risus at ipsum. Proin non porttitor neque. Vivamus fringilla ex nec iaculis cursus. Vestibulum suscipit mauris sem, vitae gravida ipsum fermentum id. Quisque pulvinar blandit ullamcorper. Donec id justo at metus scelerisque pulvinar. Proin suscipit suscipit nisi, quis tempus ipsum vulputate quis. Pellentesque sodales rutrum eros, eget pulvinar ante condimentum a. Donec accumsan, ante ut facilisis cursus, nibh quam congue eros, vitae placerat tortor magna vel lacus. Etiam odio diam, venenatis eu sollicitudin non, ultrices ut urna. Aliquam vehicula diam eu eros eleifend, ac vulputate purus faucibus.',
        165.87091900000001,
        165.871503,
        {
          type: 'Text',
          name: '5.5',
        },
      ],
      [
        'FileIO',
        174,
        175,
        {
          type: 'FileIO',
          source: 'PoisonIOInterposer',
          filename: '/foo/bar',
          operation: 'create/open',
        },
      ],
    ].sort((a, b) => a[1] - b[1])
  );
}

export function getProfileWithNamedThreads(threadNames: string[]): Profile {
  const profile = getEmptyProfile();
  profile.threads = threadNames.map((name) => getEmptyThread({ name }));
  return profile;
}

/**
 * Create a profile from text representation of samples. Each column in the text provided
 * represents a sample. Each sample is made up of a list of functions.
 * Each column needs to be separated by at least two spaces.
 *
 * Example usage:
 *
 * ```
 * const { profile } = getProfileFromTextSamples(`
 *   A             A             A                A
 *   B.js          B.js          F[cat:Graphics]  F
 *   C.js          C.js          G                G
 *   D             D                              H[cat:GC / CC]
 *   E[cat:Other]  E [cat:Other]
 * `);
 * ```
 *
 * The function names are aligned vertically on the left. This would produce 4 samples
 * with the stacks based off of those functions listed, with A being the root. Single
 * spaces within a column are permitted, surrounding whitespace is trimmed.
 *
 * Functions ending in "js" are marked as JS functions in the funcTable's isJS
 * column. Functions with categories using the notation "functionName[cat:categoryName]"
 * are annotated with that category in the frameTable's category column.
 *
 * The function returns more information as well, that is:
 * * an array mapping the func indices (IndexIntoFuncTable) to their names
 * * an array mapping the func names to their indices
 *
 * Functions ending in "js" are marked as JS functions in the funcTable's isJS
 * column. It's possible to specify a JIT type by specifying in brackets like
 * this: [jit:baseline] or [jit:ion], right after the function name (see below
 * for an example). The default is no JIT.
 *
 * The time of the sample can also be set by making the first row all numbers:
 * ```
 *   const { profile } = getProfileFromTextSamples(`
 *    0  1  5  6
 *    A  A  A  C
 *       B
 *  `);
 * ```
 *
 * The funcNamesDictPerThread array can be useful when using it like this:
 * ```
 * const {
 *   profile,
 *   funcNamesDictPerThread: [{ A, B, Cjs, D }],
 * } = getProfileFromTextSamples(`
 *    A             A
 *    B             B
 *    Cjs[jit:ion]  Cjs[jit:baseline]
 *    D[cat:DOM]    D[cat:DOM]
 *    E             F
 *  `);
 * ```
 * Now the variables named A B Cjs D directly refer to the func indices and can
 * be used in tests.
 *
 * The following func and frame attributes are supported:
 *  - [jit:*] - The JIT used for a JS frame, affects frameTable.implementation.
 *  - [cat:*] - The category name, affects frameTable.category
 *  - [lib:*] - The library name, affects funcTable.resource + resourceTable + libs
 *  - [file:*] - The filename, affects funcTable.file
 *  - [line:*] - The line, affects frameTable.line
 *  - [address:*] - The frame address, affects frameTable.address
 *  - [inl:*] - The inline depth, affects frameTable.inlineDepth

```js
// Execute the code below in the web console in the profiler to get a stack that's
// ready to be pasted into getProfileFromTextSamples.

function getFrame(
  { stackTable, frameTable, funcTable, stringTable, resourceTable, libs },
  frameIndex
) {
  const funcIndex = frameTable.func[frameIndex];
  let s = stringTable.getString(funcTable.name[funcIndex]);
  const libIndex = resourceTable.lib[funcTable.resource[funcIndex]];
  if (libIndex !== null) {
    const libName = libs[libIndex].name;
    s += `[lib:${libName}]`;
  }
  const fileStringIndex = funcTable.fileName[funcIndex];
  if (fileStringIndex !== null) {
    s += `[file:${stringTable.getString(fileStringIndex)}]`;
  }
  const line = frameTable.line[frameIndex];
  if (line !== null) {
    s += `[line:${line}]`;
  }
  const address = frameTable.address[frameIndex];
  if (address !== -1) {
    s += `[address:0x${address.toString(16)}]`;
  }
  const inlineDepth = frameTable.inlineDepth[frameIndex];
  if (inlineDepth !== 0) {
    s += `[inl:${inlineDepth}]`;
  }
  return s;
}

function getStack(thread, stackIndex) {
  const { stackTable } = thread;
  const stack = [];
  while (stackIndex !== null) {
    stack.unshift(getFrame(thread, stackTable.frame[stackIndex]));
    stackIndex = stackTable.prefix[stackIndex];
  }
  return stack;
}

getStack(filteredThread, filteredThread.samples.stack[0])
```
*/
export function getProfileFromTextSamples(...allTextSamples: string[]): {
  profile: Profile,
  funcNamesPerThread: Array<string[]>,
  funcNamesDictPerThread: Array<{ [funcName: string]: number }>,
} {
  const profile = getEmptyProfile();
  // Provide a useful marker schema, rather than an empty one.
  profile.meta.markerSchema = markerSchemaForTests;
  const categories = ensureExists(
    profile.meta.categories,
    'Expected to find categories.'
  );

  const funcNamesPerThread = [];
  const funcNamesDictPerThread = [];

  profile.threads = allTextSamples.map((textSamples) => {
    // Process the text.
    const textOnlyStacks = _parseTextSamples(textSamples);

    // See if the first row contains only numbers, if so this is the time of the sample.
    let sampleTimes = null;

    // Check if the first row is made by base 10 integers. 0x200 and other will parse
    // as numbers, but they can be used as valid function names.
    const isFirstRowMadeOfNumbers = textOnlyStacks.every((stack) =>
      /^\d+$/.test(stack[0])
    );
    if (isFirstRowMadeOfNumbers) {
      sampleTimes = textOnlyStacks.map((stack) => parseInt(stack[0]));
      for (const stack of textOnlyStacks) {
        // Remove the number.
        stack.shift();
      }
    }

    // Flatten the textOnlyStacks into into a list of function names.
    const funcNamesSet = new Set();
    const removeModifiers = /\[.*/;
    for (let i = 0; i < textOnlyStacks.length; i++) {
      const textOnlyStack = textOnlyStacks[i];
      for (let j = 0; j < textOnlyStack.length; j++) {
        funcNamesSet.add(textOnlyStack[j].replace(removeModifiers, ''));
      }
    }
    const funcNames = [...funcNamesSet];

    const funcNamesDict = funcNames.reduce((result, item, index) => {
      result[item] = index;
      return result;
    }, {});

    funcNamesPerThread.push(funcNames);
    funcNamesDictPerThread.push(funcNamesDict);

    // Turn this into a real thread.
    return _buildThreadFromTextOnlyStacks(
      textOnlyStacks,
      funcNames,
      categories,
      sampleTimes
    );
  });

  return { profile, funcNamesPerThread, funcNamesDictPerThread };
}

function _getAllMatchRanges(regex, str): Array<{ start: number, end: number }> {
  const ranges = [];

  let match;
  do {
    match = regex.exec(str);
    if (match) {
      ranges.push({ start: match.index, end: match.index + match[0].length });
    }
  } while (match);

  return ranges;
}

function _getColumnPositions(line): number[] {
  const lineWithoutIndent = line.trimLeft();
  const indent = line.length - lineWithoutIndent.length;
  const trimmedLine = line.trim();

  // Find the start and end positions of all consecutive runs of two spaces or more.
  const columnSeparatorRanges = _getAllMatchRanges(/ {2,}/g, trimmedLine);
  return [indent, ...columnSeparatorRanges.map((range) => range.end + indent)];
}

/**
 * Turn a text blob into a list of stacks.
 *
 * e.g:
 * const text = `
 *   A  A
 *   B  B
 *   C  C
 *      D
 *      E
 * `
 *
 * Returns:
 * [
 *   ['A', 'B', 'C'],
 *   ['A', 'B', 'C', 'D', E'],
 * ]
 */
function _parseTextSamples(textSamples: string): Array<string[]> {
  const lines = textSamples.split('\n').filter(
    // Filter out empty lines
    (t) => t
  );
  if (lines.length === 0) {
    throw new Error('Empty text data was sent');
  }

  // Compute the index of where the columns start in the string.
  const columnPositions = _getColumnPositions(lines[0]);

  // Create a table of string cells. Empty cells contain the empty string.
  const rows = lines.map((line) =>
    columnPositions.map((pos, columnIndex) =>
      line.substring(pos, columnPositions[columnIndex + 1]).trim()
    )
  );

  // Transpose the table to go from columns to rows.
  return columnPositions.map((_, columnIndex) => {
    const column = [];
    for (let rowIndex = 0; rowIndex < rows.length; rowIndex++) {
      const value = rows[rowIndex][columnIndex];
      if (!value) {
        break;
      }
      column.push(value);
    }
    return column;
  });
}

const JIT_IMPLEMENTATIONS = ['ion', 'baseline', 'blinterp'];

function _findJitTypeFromFuncName(funcNameWithModifier: string): string | null {
  const findJitTypeResult = /\[jit:([^\]]+)\]/.exec(funcNameWithModifier);
  let jitType = null;
  if (findJitTypeResult) {
    jitType = findJitTypeResult[1];
  }

  if (jitType) {
    if (JIT_IMPLEMENTATIONS.includes(jitType)) {
      return jitType;
    }
    throw new Error(
      `The jitType '${jitType}' is unknown to this tool. Is it a typo or should you update the list of possible values?`
    );
  }

  return null;
}

function _isJsFunctionName(funcName) {
  return funcName.endsWith('js');
}

function _findCategoryFromFuncName(
  funcNameWithModifier: string,
  funcName: string,
  categories: CategoryList
): IndexIntoCategoryList | null {
  const findCategoryResult = /\[cat:([^\]]+)\]/.exec(funcNameWithModifier);
  let categoryName;
  if (findCategoryResult) {
    categoryName = findCategoryResult[1];
  } else if (_isJsFunctionName(funcName)) {
    categoryName = 'JavaScript';
  }

  if (categoryName) {
    const category = categories.findIndex((c) => c.name === categoryName);
    if (category !== -1) {
      return category;
    }
  }

  return null;
}

function _findLibNameFromFuncName(funcNameWithModifier: string): string | null {
  const findLibNameResult = /\[lib:([^\]]+)\]/.exec(funcNameWithModifier);
  if (findLibNameResult) {
    const libName = findLibNameResult[1];
    return libName;
  }

  return null;
}

function _findFileNameFromFuncName(
  funcNameWithModifier: string
): string | null {
  const findFileNameResult = /\[file:([^\]]+)\]/.exec(funcNameWithModifier);
  if (findFileNameResult) {
    const fileName = findFileNameResult[1];
    return fileName;
  }

  return null;
}

function _findLineNumberFromFuncName(
  funcNameWithModifier: string
): LineNumber | null {
  const findLineNumberResult = /\[line:([0-9]+)\]/.exec(funcNameWithModifier);
  if (findLineNumberResult) {
    return +findLineNumberResult[1];
  }

  return null;
}

function _findAddressFromFuncName(
  funcNameWithModifier: string
): Address | null {
  const findAddressResult = /\[address:0x([0-9a-f]+)\]/.exec(
    funcNameWithModifier
  );
  if (findAddressResult) {
    return parseInt(findAddressResult[1], 16);
  }

  return null;
}

function _findInlineDepthFromFuncName(
  funcNameWithModifier: string
): Address | null {
  const findInlineDepthResult = /\[inl:([0-9]+)\]/.exec(funcNameWithModifier);
  if (findInlineDepthResult) {
    return +findInlineDepthResult[1];
  }

  return null;
}

function _buildThreadFromTextOnlyStacks(
  textOnlyStacks: Array<string[]>,
  funcNames: Array<string>,
  categories: CategoryList,
  sampleTimes: number[] | null
): Thread {
  const thread = getEmptyThread();

  const {
    funcTable,
    stringTable,
    frameTable,
    stackTable,
    samples,
    resourceTable,
    libs,
  } = thread;

  // Create the FuncTable.
  funcNames.forEach((funcName) => {
    funcTable.name.push(stringTable.indexForString(funcName));
    funcTable.fileName.push(null);
    funcTable.relevantForJS.push(funcName.endsWith('js-relevant'));
    funcTable.isJS.push(_isJsFunctionName(funcName));
    funcTable.lineNumber.push(null);
    funcTable.columnNumber.push(null);
    // Ignore resources for now, this way funcNames have really nice string indexes.
    // The resource column will be filled in the loop below.
    funcTable.length++;
  });

  const categoryOther = categories.findIndex((c) => c.name === 'Other');

  // This map caches resource indexes for library names.
  const resourceIndexCache = {};

  // Create the samples, stacks, and frames.
  textOnlyStacks.forEach((column, columnIndex) => {
    let prefix = null;
    column.forEach((funcNameWithModifier) => {
      const funcName = funcNameWithModifier.replace(/\[.*/, '');

      // There is a one-to-one relationship between strings and funcIndexes here, so
      // the indexes can double as both string indexes and func indexes.
      const funcIndex = stringTable.indexForString(funcName);

      // Find the library name from the function name and create an entry if needed.
      const libraryName = _findLibNameFromFuncName(funcNameWithModifier);
      let resourceIndex = -1;
      if (libraryName) {
        resourceIndex = resourceIndexCache[libraryName];
        if (resourceIndex === undefined) {
          libs.push({
            start: 0,
            end: 0,
            offset: 0,
            arch: '',
            name: libraryName,
            path: '/path/to/' + libraryName,
            debugName: libraryName,
            debugPath: '/path/to/' + libraryName,
            breakpadId: 'SOMETHING_FAKE',
          });

          resourceTable.lib.push(libs.length - 1); // The lastly inserted item.
          resourceTable.name.push(stringTable.indexForString(libraryName));
          resourceTable.type.push(resourceTypes.library);
          resourceTable.host.push(undefined);
          resourceIndex = resourceTable.length++;

          resourceIndexCache[libraryName] = resourceIndex;
        }
      }

      funcTable.resource[funcIndex] = resourceIndex;

      // Find the file name from the function name
      const fileName = _findFileNameFromFuncName(funcNameWithModifier);
      if (fileName) {
        funcTable.fileName[funcIndex] = stringTable.indexForString(fileName);
      }

      // Find the wanted jit type from the function name
      const jitType = _findJitTypeFromFuncName(funcNameWithModifier);
      const jitTypeIndex = jitType ? stringTable.indexForString(jitType) : null;
      const category = _findCategoryFromFuncName(
        funcNameWithModifier,
        funcName,
        categories
      );
      const lineNumber = _findLineNumberFromFuncName(funcNameWithModifier);
      const address =
        _findAddressFromFuncName(funcNameWithModifier) ??
        (funcName.startsWith('0x') ? parseInt(funcName.substr(2), 16) : -1);
      const inlineDepth =
        _findInlineDepthFromFuncName(funcNameWithModifier) ?? 0;

      // Attempt to find a frame that satisfies the given funcIndex, jit type,
      // category, and line number.
      let frameIndex;
      for (let i = 0; i < frameTable.length; i++) {
        if (
          funcIndex === frameTable.func[i] &&
          jitTypeIndex === frameTable.implementation[i] &&
          category === frameTable.category[i] &&
          lineNumber === frameTable.line[i] &&
          address === frameTable.address[i] &&
          inlineDepth === frameTable.inlineDepth[i]
        ) {
          frameIndex = i;
          break;
        }
      }

      if (frameIndex === undefined) {
        frameTable.func.push(funcIndex);
        frameTable.address.push(address);
        frameTable.inlineDepth.push(inlineDepth);
        frameTable.category.push(category);
        frameTable.subcategory.push(0);
        frameTable.innerWindowID.push(0);
        frameTable.nativeSymbol.push(null);
        frameTable.implementation.push(jitTypeIndex);
        frameTable.line.push(lineNumber);
        frameTable.column.push(null);
        frameTable.optimizations.push(null);
        frameIndex = frameTable.length++;
      }

      // Attempt to find a stack that satisfies the given frameIndex and prefix.
      let stackIndex;
      for (let i = 0; i < stackTable.length; i++) {
        if (
          stackTable.prefix[i] === prefix &&
          stackTable.frame[i] === frameIndex
        ) {
          stackIndex = i;
          break;
        }
      }

      // If we couldn't find a stack, go ahead and create it.
      if (stackIndex === undefined) {
        const frameCategory = frameTable.category[frameIndex];
        const frameSubcategory = frameTable.subcategory[frameIndex];
        const prefixCategory =
{'fraction_non_alphanumeric': 0.11213511815221738, 'fraction_numerical': 0.00683588171449245, 'mean_word_length': 2.8864796862280766, 'pattern_counts': {'":': 0, '<': 31, '<?xml version=': 0, '>': 57, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 4, 'word_list_counts': {'cursed_substrings.json': 10, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4325081', 'n_tokens_mistral': 16198, 'n_tokens_neox': 15722, 'n_words': 4294}