<commit_before>/* This Source Code Form is subject to the terms of the Mozilla Public
 * License, v. 2.0. If a copy of the MPL was not distributed with this
 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */

// @flow
import {
  getEmptyProfile,
  getEmptyThread,
  getEmptyJsTracerTable,
} from '../../../profile-logic/data-structures';
import { UniqueStringArray } from '../../../utils/unique-string-array';

import type {
  Profile,
  Thread,
  ThreadIndex,
  IndexIntoCategoryList,
  CategoryList,
  JsTracerTable,
  Counter,
} from '../../../types/profile';
import type { MarkerPayload, NetworkPayload } from '../../../types/markers';
import type { Milliseconds } from '../../../types/units';

// Array<[MarkerName, Milliseconds, Data]>
type MarkerName = string;
type MarkerTime = Milliseconds;
type MockPayload = {| startTime: Milliseconds, endTime: Milliseconds |};
type TestDefinedMarkers = Array<
  [MarkerName, MarkerTime, MarkerPayload | MockPayload]
>;
export type TestDefinedJsTracerEvent = [
  // Event name:
  string,
  // Start time:
  Milliseconds,
  // End time:
  Milliseconds,
];

/**
 * This function ensures that the mock payloads are converted correctly to real payloads
 * that match the MarkerPayload typing. Specifically it adds the 'DummyForTests' type
 * to { startTime, endTime } payloads. Doing this means that it's cleaner to create
 * dummy test-defined markers, since we don't have to add a `type` to the payload.
 */
function _refineMockPayload(
  payload: MarkerPayload | MockPayload
): MarkerPayload {
  if (
    // Check for a MockPayload.
    payload !== null &&
    Object.keys(payload).length === 2 &&
    typeof payload.startTime === 'number' &&
    typeof payload.endTime === 'number'
  ) {
    return {
      type: 'DummyForTests',
      endTime: payload.endTime,
      startTime: payload.startTime,
    };
  }
  // There is no way to refine the payload type to just the { startTime, endTime }
  // mock marker. So check for those conditions above, and coerce the final result
  // into a MarkerPayload using the function signature.
  return (payload: any);
}

export function addMarkersToThreadWithCorrespondingSamples(
  thread: Thread,
  markers: TestDefinedMarkers
) {
  const stringTable = thread.stringTable;
  const markersTable = thread.markers;
  const samples = thread.samples;

  markers.forEach(([name, time, data]) => {
    markersTable.name.push(stringTable.indexForString(name));
    markersTable.time.push(time);
    markersTable.data.push(_refineMockPayload(data));
    markersTable.length++;

    // Try to get a consistent profile with a sample for each marker.
    const startTime = time;
    // If we have no data, endTime is the same as startTime.
    const endTime =
      data && typeof data.endTime === 'number' ? data.endTime : time;

    // Push on the start and end time if necessary
    [startTime, endTime].forEach(time => {
      if (!samples.time.includes(time)) {
        samples.time.push(time);
        samples.stack.push(null);
        samples.rss.push(null);
        samples.uss.push(null);
        samples.responsiveness.push(null);
        samples.length++;
      }
    });
  });

  samples.time.sort();
}

export function getThreadWithMarkers(markers: TestDefinedMarkers) {
  const thread = getEmptyThread();
  addMarkersToThreadWithCorrespondingSamples(thread, markers);
  return thread;
}

export function getProfileWithMarkers(
  ...markersPerThread: TestDefinedMarkers[]
): Profile {
  const profile = getEmptyProfile();
  profile.threads = markersPerThread.map(testDefinedMarkers =>
    getThreadWithMarkers(testDefinedMarkers)
  );
  return profile;
}

export function getProfileWithNamedThreads(threadNames: string[]): Profile {
  const profile = getEmptyProfile();
  profile.threads = threadNames.map(name => getEmptyThread({ name }));
  return profile;
}

/**
 * Create a profile from text representation of samples. Each column in the text provided
 * represents a sample. Each sample is made up of a list of functions.
 * Each column needs to be separated by at least two spaces.
 *
 * Example usage:
 *
 * ```
 * const { profile } = getProfileFromTextSamples(`
 *   A             A             A                A
 *   B.js          B.js          F[cat:Graphics]  F
 *   C.js          C.js          G                G
 *   D             D                              H[cat:GC / CC]
 *   E[cat:Other]  E [cat:Other]
 * `);
 * ```
 *
 * The function names are aligned vertically on the left. This would produce 4 samples
 * with the stacks based off of those functions listed, with A being the root. Single
 * spaces within a column are permitted, surrounding whitespace is trimmed.
 *
 * Functions ending in "js" are marked as JS functions in the funcTable's isJS
 * column. Functions with categories using the notation "functionName[cat:categoryName]"
 * are annotated with that category in the frameTable's category column.
 *
 * The function returns more information as well, that is:
 * * an array mapping the func indices (IndexIntoFuncTable) to their names
 * * an array mapping the func names to their indices
 *
 * Functions ending in "js" are marked as JS functions in the funcTable's isJS
 * column. It's possible to specify a JIT type by specifying in brackets like
 * this: [jit:baseline] or [jit:ion], right after the function name (see below
 * for an example). The default is no JIT.
 *
 * The funcNamesDictPerThread array can be useful when using it like this:
 * ```
 * const {
 *   profile,
 *   funcNamesDictPerThread: [{ A, B, Cjs, D }],
 * } = getProfileFromTextSamples(`
 *    A             A
 *    B             B
 *    Cjs[jit:ion]  Cjs[jit:baseline]
 *    D[cat:DOM]    D[cat:DOM]
 *    E             F
 *  `);
 * ```
 * Now the variables named A B Cjs D directly refer to the func indices and can
 * be used in tests.
 */
export function getProfileFromTextSamples(
  ...allTextSamples: string[]
): {
  profile: Profile,
  funcNamesPerThread: Array<string[]>,
  funcNamesDictPerThread: Array<{ [funcName: string]: number }>,
} {
  const profile = getEmptyProfile();
  const categories = profile.meta.categories;

  const funcNamesPerThread = [];
  const funcNamesDictPerThread = [];

  profile.threads = allTextSamples.map(textSamples => {
    // Process the text.
    const textOnlyStacks = _parseTextSamples(textSamples);
    const funcNames = textOnlyStacks
      // Flatten the arrays.
      .reduce((memo, row) => [...memo, ...row], [])
      // remove modifiers
      .map(func => func.replace(/\[.*/, ''))
      // Make the list unique.
      .filter((item, index, array) => array.indexOf(item) === index);

    const funcNamesDict = funcNames.reduce((result, item, index) => {
      result[item] = index;<commit_msg>Make the tests run faster by speeding up funcNames in getProfileFromTextSamples (Merge PR #1658)<commit_after>/* This Source Code Form is subject to the terms of the Mozilla Public
 * License, v. 2.0. If a copy of the MPL was not distributed with this
 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */

// @flow
import {
  getEmptyProfile,
  getEmptyThread,
  getEmptyJsTracerTable,
} from '../../../profile-logic/data-structures';
import { UniqueStringArray } from '../../../utils/unique-string-array';

import type {
  Profile,
  Thread,
  ThreadIndex,
  IndexIntoCategoryList,
  CategoryList,
  JsTracerTable,
  Counter,
} from '../../../types/profile';
import type { MarkerPayload, NetworkPayload } from '../../../types/markers';
import type { Milliseconds } from '../../../types/units';

// Array<[MarkerName, Milliseconds, Data]>
type MarkerName = string;
type MarkerTime = Milliseconds;
type MockPayload = {| startTime: Milliseconds, endTime: Milliseconds |};
type TestDefinedMarkers = Array<
  [MarkerName, MarkerTime, MarkerPayload | MockPayload]
>;
export type TestDefinedJsTracerEvent = [
  // Event name:
  string,
  // Start time:
  Milliseconds,
  // End time:
  Milliseconds,
];

/**
 * This function ensures that the mock payloads are converted correctly to real payloads
 * that match the MarkerPayload typing. Specifically it adds the 'DummyForTests' type
 * to { startTime, endTime } payloads. Doing this means that it's cleaner to create
 * dummy test-defined markers, since we don't have to add a `type` to the payload.
 */
function _refineMockPayload(
  payload: MarkerPayload | MockPayload
): MarkerPayload {
  if (
    // Check for a MockPayload.
    payload !== null &&
    Object.keys(payload).length === 2 &&
    typeof payload.startTime === 'number' &&
    typeof payload.endTime === 'number'
  ) {
    return {
      type: 'DummyForTests',
      endTime: payload.endTime,
      startTime: payload.startTime,
    };
  }
  // There is no way to refine the payload type to just the { startTime, endTime }
  // mock marker. So check for those conditions above, and coerce the final result
  // into a MarkerPayload using the function signature.
  return (payload: any);
}

export function addMarkersToThreadWithCorrespondingSamples(
  thread: Thread,
  markers: TestDefinedMarkers
) {
  const stringTable = thread.stringTable;
  const markersTable = thread.markers;
  const samples = thread.samples;

  markers.forEach(([name, time, data]) => {
    markersTable.name.push(stringTable.indexForString(name));
    markersTable.time.push(time);
    markersTable.data.push(_refineMockPayload(data));
    markersTable.length++;

    // Try to get a consistent profile with a sample for each marker.
    const startTime = time;
    // If we have no data, endTime is the same as startTime.
    const endTime =
      data && typeof data.endTime === 'number' ? data.endTime : time;

    // Push on the start and end time if necessary
    [startTime, endTime].forEach(time => {
      if (!samples.time.includes(time)) {
        samples.time.push(time);
        samples.stack.push(null);
        samples.rss.push(null);
        samples.uss.push(null);
        samples.responsiveness.push(null);
        samples.length++;
      }
    });
  });

  samples.time.sort();
}

export function getThreadWithMarkers(markers: TestDefinedMarkers) {
  const thread = getEmptyThread();
  addMarkersToThreadWithCorrespondingSamples(thread, markers);
  return thread;
}

export function getProfileWithMarkers(
  ...markersPerThread: TestDefinedMarkers[]
): Profile {
  const profile = getEmptyProfile();
  profile.threads = markersPerThread.map(testDefinedMarkers =>
    getThreadWithMarkers(testDefinedMarkers)
  );
  return profile;
}

export function getProfileWithNamedThreads(threadNames: string[]): Profile {
  const profile = getEmptyProfile();
  profile.threads = threadNames.map(name => getEmptyThread({ name }));
  return profile;
}

/**
 * Create a profile from text representation of samples. Each column in the text provided
 * represents a sample. Each sample is made up of a list of functions.
 * Each column needs to be separated by at least two spaces.
 *
 * Example usage:
 *
 * ```
 * const { profile } = getProfileFromTextSamples(`
 *   A             A             A                A
 *   B.js          B.js          F[cat:Graphics]  F
 *   C.js          C.js          G                G
 *   D             D                              H[cat:GC / CC]
 *   E[cat:Other]  E [cat:Other]
 * `);
 * ```
 *
 * The function names are aligned vertically on the left. This would produce 4 samples
 * with the stacks based off of those functions listed, with A being the root. Single
 * spaces within a column are permitted, surrounding whitespace is trimmed.
 *
 * Functions ending in "js" are marked as JS functions in the funcTable's isJS
 * column. Functions with categories using the notation "functionName[cat:categoryName]"
 * are annotated with that category in the frameTable's category column.
 *
 * The function returns more information as well, that is:
 * * an array mapping the func indices (IndexIntoFuncTable) to their names
 * * an array mapping the func names to their indices
 *
 * Functions ending in "js" are marked as JS functions in the funcTable's isJS
 * column. It's possible to specify a JIT type by specifying in brackets like
 * this: [jit:baseline] or [jit:ion], right after the function name (see below
 * for an example). The default is no JIT.
 *
 * The funcNamesDictPerThread array can be useful when using it like this:
 * ```
 * const {
 *   profile,
 *   funcNamesDictPerThread: [{ A, B, Cjs, D }],
 * } = getProfileFromTextSamples(`
 *    A             A
 *    B             B
 *    Cjs[jit:ion]  Cjs[jit:baseline]
 *    D[cat:DOM]    D[cat:DOM]
 *    E             F
 *  `);
 * ```
 * Now the variables named A B Cjs D directly refer to the func indices and can
 * be used in tests.
 */
export function getProfileFromTextSamples(
  ...allTextSamples: string[]
): {
  profile: Profile,
  funcNamesPerThread: Array<string[]>,
  funcNamesDictPerThread: Array<{ [funcName: string]: number }>,
} {
  const profile = getEmptyProfile();
  const categories = profile.meta.categories;

  const funcNamesPerThread = [];
  const funcNamesDictPerThread = [];

  profile.threads = allTextSamples.map(textSamples => {
    // Process the text.
    const textOnlyStacks = _parseTextSamples(textSamples);

    // Flatten the textOnlyStacks into into a list of function names.
    const funcNamesSet = new Set();
    const removeModifiers = /\[.*/;
    for (let i = 0; i < textOnlyStacks.length; i++) {
      const textOnlyStack = textOnlyStacks[i];
      for (let j = 0; j < textOnlyStack.length; j++) {
        funcNamesSet.add(textOnlyStack[j].replace(removeModifiers, ''));
      }
    }
    const funcNames = [...funcNamesSet];

    const funcNamesDict = funcNames.reduce((result, item, index) => {
      result[item] = index;
{'fraction_non_alphanumeric': 0.11005098324836125, 'fraction_numerical': 0.0013109978150036416, 'mean_word_length': 3.137089484784574, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 26, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '667911', 'n_tokens_mistral': 4031, 'n_tokens_neox': 3915, 'n_words': 1324}