<commit_before>import { pick } from "lodash";
import { measurementIdSymbol, measurementJitterSymbol } from "../util/globals";
import {
  APPLY_MEASUREMENTS_FILTER,
  CHANGE_MEASUREMENTS_COLLECTION,
  LOAD_MEASUREMENTS
} from "./types";

/**
 * Find the collection within collections that has a key matching the provided
 * collectionKey.
 *
 * If collectionKey is not provided, returns the first collection.
 * If no matches are found, returns the first collection.
 * If multiple matches are found, only returns the first matching collection.
 *
 * @param {Array<Object>} collections
 * @param {string} collectionKey
 * @returns {Object}
 */
export const getCollectionToDisplay = (collections, collectionKey) => {
  if (!collectionKey) return collections[0];
  const potentialCollections = collections.filter((collection) => collection.key === collectionKey);
  if (potentialCollections.length === 0) return collections[0];
  if (potentialCollections.length > 1) {
    console.error(`Found multiple collections with key ${collectionKey}. Returning the first matching collection only.`);
  }
  return potentialCollections[0];
};

/**
 * Constructs the controls redux state for the measurements panel based on
 * config values within the provided collection.
 *
 * If no display defaults are provided, uses the current controls redux state.
 * If the current `measurementsGrouping` does not exist in the collection, then
 * defaults to the first grouping option.
 * @param {Object} collection
 * @returns {Object}
 */
const getCollectionDisplayControls = (controls, collection) => {
  const measurementsControls = [
    "measurementsGroupBy",
    "measurementsDisplay",
    "measurementsShowOverallMean",
    "measurementsShowThreshold",
    "measurementsFilters"
  ];
  // Copy current control options for measurements
  const newControls = pick(controls, measurementsControls);
  // Checks the current group by is available as a field in collection
  if (!collection.fields.has(newControls.measurementsGroupBy)) {
    // If current group by is not available as a field, then default to the first grouping option.
    newControls.measurementsGroupBy = collection.groupings[0];
  }

  // Verify that current filters are valid for the new collection
  Object.entries(newControls.measurementsFilters).forEach(([field, valuesMap]) => {
    // Delete filter for field that does not exist in the new collection filters
    if (!collection.filters.has(field)) {
      return delete newControls.measurementsFilters[field];
    }
    // Clone nested Map to avoid changing redux state in place
    newControls.measurementsFilters[field] = new Map(valuesMap);
    return [...valuesMap.keys()].forEach((value) => {
      // Delete filter for values that do not exist within the field of the new collection
      if (!collection.filters.get(field).values.has(value)) {
        newControls.measurementsFilters[field].delete(value);
      }
    });
  });

  if (collection["display_defaults"]) {
    const {
      group_by,
      measurement_display,
      show_overall_mean,
      show_threshold
    } = collection["display_defaults"];

    if (group_by) {
      newControls.measurementsGroupBy = group_by;
    }
    if (measurement_display) {
      newControls.measurementsDisplay = measurement_display;
    }
    if (typeof show_overall_mean === "boolean") {
      newControls.measurementsShowOverallMean = show_overall_mean;
    }
    if (typeof show_threshold === "boolean") {
      newControls.measurementsShowThreshold = show_threshold;
    }
  }

  return newControls;
};

export const loadMeasurements = (json) => (dispatch, getState) => {
  // TODO: Load controls from state to get potential url query parameters
  const { tree, controls } = getState();
  if (!tree.loaded) {
    throw new Error("tree not loaded");
  }

  const collections = json["collections"];
  if (!collections || collections.length === 0) {
    throw new Error("Measurements JSON does not have collections");
  }

  collections.forEach((collection) => {
    /*
     * Create fields Map for easier access of titles and to keep ordering
     * First add fields from JSON to keep user's ordering
     * Then loop over measurements to add any remaining fields
     */
    const collectionFieldsArray = collection.fields;
    collection.fields = new Map();
    if (collectionFieldsArray && collectionFieldsArray.length > 0) {
      collectionFieldsArray.forEach(({key, title}) => {
        collection.fields.set(key, {title: title || key});
      });
    }

    /**
     * Create filters Map for easier access of values and to keep ordering
     * First add fields from JSON to keep user's ordering
     * Then loop over measurements to add values
     * If there are no JSON defined filters, then add all fields as filters
     */
    const collectionFiltersArray = collection.filters;
    collection.filters = new Map();
    if (collectionFiltersArray && collectionFiltersArray.length > 0) {
      collectionFiltersArray.forEach((filterField) => {
        collection.filters.set(filterField, {values: new Set()});
      });
    }

    collection.measurements.forEach((measurement, index) => {
      Object.entries(measurement).forEach(([field, fieldValue]) => {
        // Add remaining field titles
        if (!collection.fields.has(field)) {
          collection.fields.set(field, {title: field});
        }

        // Only save the unique values if the field is in defined filters
        // OR there are no JSON defined filters, so all fields are filters
        if ((collection.filters.has(field)) || !collectionFiltersArray) {
          const filterObject = collection.filters.get(field) || { values: new Set()};<commit_msg>Measurements: refactor new Map creations<commit_after>import { pick } from "lodash";
import { measurementIdSymbol, measurementJitterSymbol } from "../util/globals";
import {
  APPLY_MEASUREMENTS_FILTER,
  CHANGE_MEASUREMENTS_COLLECTION,
  LOAD_MEASUREMENTS
} from "./types";

/**
 * Find the collection within collections that has a key matching the provided
 * collectionKey.
 *
 * If collectionKey is not provided, returns the first collection.
 * If no matches are found, returns the first collection.
 * If multiple matches are found, only returns the first matching collection.
 *
 * @param {Array<Object>} collections
 * @param {string} collectionKey
 * @returns {Object}
 */
export const getCollectionToDisplay = (collections, collectionKey) => {
  if (!collectionKey) return collections[0];
  const potentialCollections = collections.filter((collection) => collection.key === collectionKey);
  if (potentialCollections.length === 0) return collections[0];
  if (potentialCollections.length > 1) {
    console.error(`Found multiple collections with key ${collectionKey}. Returning the first matching collection only.`);
  }
  return potentialCollections[0];
};

/**
 * Constructs the controls redux state for the measurements panel based on
 * config values within the provided collection.
 *
 * If no display defaults are provided, uses the current controls redux state.
 * If the current `measurementsGrouping` does not exist in the collection, then
 * defaults to the first grouping option.
 * @param {Object} collection
 * @returns {Object}
 */
const getCollectionDisplayControls = (controls, collection) => {
  const measurementsControls = [
    "measurementsGroupBy",
    "measurementsDisplay",
    "measurementsShowOverallMean",
    "measurementsShowThreshold",
    "measurementsFilters"
  ];
  // Copy current control options for measurements
  const newControls = pick(controls, measurementsControls);
  // Checks the current group by is available as a field in collection
  if (!collection.fields.has(newControls.measurementsGroupBy)) {
    // If current group by is not available as a field, then default to the first grouping option.
    newControls.measurementsGroupBy = collection.groupings[0];
  }

  // Verify that current filters are valid for the new collection
  Object.entries(newControls.measurementsFilters).forEach(([field, valuesMap]) => {
    // Delete filter for field that does not exist in the new collection filters
    if (!collection.filters.has(field)) {
      return delete newControls.measurementsFilters[field];
    }
    // Clone nested Map to avoid changing redux state in place
    newControls.measurementsFilters[field] = new Map(valuesMap);
    return [...valuesMap.keys()].forEach((value) => {
      // Delete filter for values that do not exist within the field of the new collection
      if (!collection.filters.get(field).values.has(value)) {
        newControls.measurementsFilters[field].delete(value);
      }
    });
  });

  if (collection["display_defaults"]) {
    const {
      group_by,
      measurement_display,
      show_overall_mean,
      show_threshold
    } = collection["display_defaults"];

    if (group_by) {
      newControls.measurementsGroupBy = group_by;
    }
    if (measurement_display) {
      newControls.measurementsDisplay = measurement_display;
    }
    if (typeof show_overall_mean === "boolean") {
      newControls.measurementsShowOverallMean = show_overall_mean;
    }
    if (typeof show_threshold === "boolean") {
      newControls.measurementsShowThreshold = show_threshold;
    }
  }

  return newControls;
};

export const loadMeasurements = (json) => (dispatch, getState) => {
  // TODO: Load controls from state to get potential url query parameters
  const { tree, controls } = getState();
  if (!tree.loaded) {
    throw new Error("tree not loaded");
  }

  const collections = json["collections"];
  if (!collections || collections.length === 0) {
    throw new Error("Measurements JSON does not have collections");
  }

  collections.forEach((collection) => {
    /*
     * Create fields Map for easier access of titles and to keep ordering
     * First add fields from JSON to keep user's ordering
     * Then loop over measurements to add any remaining fields
     */
    collection.fields = new Map(
      (collection.fields || [])
        .map(({key, title}) => [key, {title: title || key}])
    );

    /**
     * Create filters Map for easier access of values and to keep ordering
     * First add fields from JSON to keep user's ordering
     * Then loop over measurements to add values
     * If there are no JSON defined filters, then add all fields as filters
     */
    const collectionFiltersArray = collection.filters;
    collection.filters = new Map(
      (collection.filters || [])
        .map((filterField) => [filterField, {values: new Set()}])
    );

    collection.measurements.forEach((measurement, index) => {
      Object.entries(measurement).forEach(([field, fieldValue]) => {
        // Add remaining field titles
        if (!collection.fields.has(field)) {
          collection.fields.set(field, {title: field});
        }

        // Only save the unique values if the field is in defined filters
        // OR there are no JSON defined filters, so all fields are filters
        if ((collection.filters.has(field)) || !collectionFiltersArray) {
          const filterObject = collection.filters.get(field) || { values: new Set()};
{'fraction_non_alphanumeric': 0.10052862646716244, 'fraction_numerical': 0.0014335633007795, 'mean_word_length': 3.627694859038143, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 33, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7192187', 'n_tokens_mistral': 2885, 'n_tokens_neox': 2855, 'n_words': 976}