<commit_before>import SECT_VALUES from './../constants/sectValues';
import CATEGORY_VALUES from './../constants/categoryValues';

const reArr = ['target-entry', 'explaination', 'original', 'source'].map((prop) => {
  return {
    prop,
    re: new RegExp(`^${prop}-(.+)$`)
  };
});

export default function entryKeysToDataRows(keys, data, f) {

  return keys.map((key) => {

    const value = data[key];

    for (let i = 0; i < reArr.length; i++) {
      const {prop, re} = reArr[i];
      const lang = (re.exec(key) || [])[1];
      if (lang) {
        return {key: prop, value, lang};
      }
    }

    if ('category' === key) {
      const categoryId = CATEGORY_VALUES.find((row) => row.value === value).id;
      return {key, value: f(categoryId)};
    }

    if ('sect' === key) {
      const sectId = SECT_VALUES.find((row) => row.value === value).id;
      return {key, value: f(sectId)};
    }
    return {key, value};
  });
}
<commit_msg>check valid langs<commit_after>import SECT_VALUES from './../constants/sectValues';
import CATEGORY_VALUES from './../constants/categoryValues';
import DICTIONARY_LANGS from './../constants/dictionaryLangs';

const validLangs = DICTIONARY_LANGS.map((row) => row.value);

const reArr = ['target-entry', 'explaination', 'explaination-source', 'explaination-note', 'original'].map((prop) => {
  return {
    prop,
    re: new RegExp(`^${prop}-(.+)$`)
  };
});

export default function entryKeysToDataRows(keys, data, f) {

  return keys.map((key) => {

    const value = data[key];

    for (let i = 0; i < reArr.length; i++) {
      const {prop, re} = reArr[i];
      const lang = (re.exec(key) || [])[1];
      if (validLangs.includes(lang)) {
        return {key: prop, value, lang};
      }
    }

    if ('category' === key) {
      const categoryId = CATEGORY_VALUES.find((row) => row.value === value).id;
      return {key, value: f(categoryId)};
    }

    if ('sect' === key) {
      const sectId = SECT_VALUES.find((row) => row.value === value).id;
      return {key, value: f(sectId)};
    }
    return {key, value};
  });
}

{'fraction_non_alphanumeric': 0.19548511047070125, 'fraction_numerical': 0.0019212295869356388, 'mean_word_length': 2.9904214559386975, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 12, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5812387', 'n_tokens_mistral': 736, 'n_tokens_neox': 727, 'n_words': 169}