<commit_before>import { rgb } from "d3-color";
import { errorNotification, successNotification, warningNotification } from "../notifications";
import { ADD_EXTRA_METADATA } from "../types";
import { parseCsvTsv } from "./parseCsvTsv";


const handleMetadata = async (dispatch, getState, file) => {
  const fileName = file.name;

  try {
    /* Parse & interrogate the CSV file */
    const {errors, data, meta} = await parseCsvTsv(file);
    if (errors.length) {
      console.error(errors);
      throw new Error(errors.map((e) => e.message).join(", "));
    }

    const {coloringInfo, strainKey, ignoredFields} = processHeader(meta.fields);
    const rows = {};
    data.forEach((d) => {rows[d[strainKey]]=d;});

    /* For each coloring, extract values defined in each row etc */
    const newNodeAttrs = {};
    const newColorings = {};
    processColorings(newNodeAttrs, newColorings, coloringInfo, rows, fileName); // modifies `newNodeAttrs` and `newColorings`

    /* TODO - process lat/longs, if present */

    /* Fix errors in data & dispatch warnings here, as we cannot dispatch in the reducers */
    const ok = checkDataForErrors(dispatch, getState, newNodeAttrs, newColorings, ignoredFields, fileName);
    if (!ok) return undefined;

    dispatch({type: ADD_EXTRA_METADATA, newColorings, newNodeAttrs});
    return dispatch(successNotification({
      message: `Adding metadata from ${fileName}`,
      details: `${Object.keys(newColorings).length} new coloring${Object.keys(newColorings).length > 1 ? "s" : ""} for ${Object.keys(newNodeAttrs).length} node${Object.keys(newNodeAttrs).length > 1 ? "s" : ""}`
    }));
  } catch (err) {
    return dispatch(errorNotification({
      message: `Parsing of ${fileName} failed`,
      details: err.message
    }));
  }
};

export default handleMetadata;


/* ---------------------- helper functions to parse data ---------------------------- */


function processHeader(fields) {
  const strainKey = fields[0];

  /* There are a number of "special case" columns we currently ignore */
  const fieldsToIgnore = new Set(["name", "div", "vaccine", "labels", "hidden", "mutations", "url", "authors", "accession", "traits", "children"]);
  fieldsToIgnore.add("num_date").add("year").add("month").add("date"); /* TODO - implement date parsing */
  fieldsToIgnore.add("latitude").add("longitude"); /* TODO - implement lat/long parsing */
  const ignoredFields = new Set();

  const coloringInfo = fields.slice(1)
    .map((fieldName) => {
      if (fieldsToIgnore.has(fieldName)) {
        ignoredFields.add(fieldName);
        return null;
      }
      let name = fieldName;
      const lookupKey = fieldName;
      const scaleType = "categorical"; // TODO
      /* interpret column names using microreact-style syntax */
      if (fieldName.includes("__")) {
        const [prefix, suffix] = fieldName.split("__");
        if (["shape", "colour"].includes(suffix)) {
          ignoredFields.add(fieldName);
          return null;
        }
        if (suffix === "autocolour") {
          name = prefix; /* MicroReact uses this to colour things, but we do this by default */
        }
      }
      return {name, lookupKey, scaleKey: undefined, scaleType};
    })
    .filter((x) => !!x)
    .map((data) => {
      if (fields.includes(`${data.name}__colour`)) {
        data.scaleKey = `${data.name}__colour`;
      }
      return data;
    });
  return {coloringInfo, header: fields, strainKey, ignoredFields};
}

/**
 * Add colorings defined by the CSV header (`coloringInfo`) and specified in each CSV<commit_msg>[metadata csv] allow colour and color spellings<commit_after>import { rgb } from "d3-color";
import { errorNotification, successNotification, warningNotification } from "../notifications";
import { ADD_EXTRA_METADATA } from "../types";
import { parseCsvTsv } from "./parseCsvTsv";


const handleMetadata = async (dispatch, getState, file) => {
  const fileName = file.name;

  try {
    /* Parse & interrogate the CSV file */
    const {errors, data, meta} = await parseCsvTsv(file);
    if (errors.length) {
      console.error(errors);
      throw new Error(errors.map((e) => e.message).join(", "));
    }

    const {coloringInfo, strainKey, ignoredFields} = processHeader(meta.fields);
    const rows = {};
    data.forEach((d) => {rows[d[strainKey]]=d;});

    /* For each coloring, extract values defined in each row etc */
    const newNodeAttrs = {};
    const newColorings = {};
    processColorings(newNodeAttrs, newColorings, coloringInfo, rows, fileName); // modifies `newNodeAttrs` and `newColorings`

    /* TODO - process lat/longs, if present */

    /* Fix errors in data & dispatch warnings here, as we cannot dispatch in the reducers */
    const ok = checkDataForErrors(dispatch, getState, newNodeAttrs, newColorings, ignoredFields, fileName);
    if (!ok) return undefined;

    dispatch({type: ADD_EXTRA_METADATA, newColorings, newNodeAttrs});
    return dispatch(successNotification({
      message: `Adding metadata from ${fileName}`,
      details: `${Object.keys(newColorings).length} new coloring${Object.keys(newColorings).length > 1 ? "s" : ""} for ${Object.keys(newNodeAttrs).length} node${Object.keys(newNodeAttrs).length > 1 ? "s" : ""}`
    }));
  } catch (err) {
    return dispatch(errorNotification({
      message: `Parsing of ${fileName} failed`,
      details: err.message
    }));
  }
};

export default handleMetadata;


/* ---------------------- helper functions to parse data ---------------------------- */


function processHeader(fields) {
  const strainKey = fields[0];

  /* There are a number of "special case" columns we currently ignore */
  const fieldsToIgnore = new Set(["name", "div", "vaccine", "labels", "hidden", "mutations", "url", "authors", "accession", "traits", "children"]);
  fieldsToIgnore.add("num_date").add("year").add("month").add("date"); /* TODO - implement date parsing */
  fieldsToIgnore.add("latitude").add("longitude"); /* TODO - implement lat/long parsing */
  const ignoredFields = new Set();

  const coloringInfo = fields.slice(1)
    .map((fieldName) => {
      if (fieldsToIgnore.has(fieldName)) {
        ignoredFields.add(fieldName);
        return null;
      }
      let name = fieldName;
      const lookupKey = fieldName;
      const scaleType = "categorical"; // TODO
      /* interpret column names using microreact-style syntax */
      if (fieldName.includes("__")) {
        const [prefix, suffix] = fieldName.split("__");
        if (["shape", "colour", "color"].includes(suffix)) {
          ignoredFields.add(fieldName);
          return null;
        }
        if (suffix === "autocolour") {
          name = prefix; /* MicroReact uses this to colour things, but we do this by default */
        }
      }
      return {name, lookupKey, scaleKey: undefined, scaleType};
    })
    .filter((x) => !!x)
    .map((data) => {
      if (fields.includes(`${data.name}__colour`)) {
        data.scaleKey = `${data.name}__colour`;
      } else if (fields.includes(`${data.name}__color`)) {
        data.scaleKey = `${data.name}__color`;
      }
      return data;
    });
  return {coloringInfo, header: fields, strainKey, ignoredFields};
}

/**
 * Add colorings defined by the CSV header (`coloringInfo`) and specified in each CSV
{'fraction_non_alphanumeric': 0.15946798299739476, 'fraction_numerical': 0.001371177841766077, 'mean_word_length': 3.1919540229885057, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5547080', 'n_tokens_mistral': 2154, 'n_tokens_neox': 2106, 'n_words': 577}