<commit_before>import path from 'path';
import { find, isEmpty, omit, cloneDeep } from 'lodash';
import fs from 'fs';

import getSectionsTree from './get-sections-tree';
import { ymlToJson, jsonToYml } from './utils';


function parseTranslation(json, rootFolder) {
  const { children } = getSectionsTree(rootFolder);
  const manifestFile = find(children, { extension: 'json' });


  if (manifestFile) {
    const manifest = JSON.parse(manifestFile.content);
    const newJson = {};

    for (const key in manifest) {
      if (manifest.hasOwnProperty(key)) {
        const val = manifest[key];
        const matchExample = val.match(/{{(.*)}}/);

        if ( ! isEmpty(matchExample)) {
          // add to new json
          Object.assign(newJson, {
            [key]: val,
          });

          const folder = matchExample[1];
          parseTranslation(json[key], `${rootFolder}/${folder}`);
          json = omit(json, key);   // deletes the folder key so that we are
                                    // left with only with simple keys
        }

        else {
          // we have to check if the key is still in the json, if it is replace
          // the contents
          if (json[key]) {
            Object.assign(newJson, {
              [key]: json[key],
            });

            json = omit(json, key);
          }
        }
      }
    }

    // now we are left only with simple keys, so add them to the manifest (newJson)
    Object.assign(newJson, json);
    fs.writeFile(`${rootFolder}/manifest.json`, JSON.stringify(newJson, null, 2), (err) => {
      if (err) throw err;
    });<commit_msg>small cleanup<commit_after>import path from 'path';
import { find, isEmpty, omit, cloneDeep } from 'lodash';
import fs from 'fs';

import getSectionsTree from './get-sections-tree';
import { ymlToJson, jsonToYml } from './utils';


function parseTranslation(json, rootFolder) {
  const { children } = getSectionsTree(rootFolder);
  const manifestFile = find(children, { extension: 'json' });

  if (manifestFile) {
    const manifest = JSON.parse(manifestFile.content);
    const newJson = {};

    for (const key in manifest) {
      if (manifest.hasOwnProperty(key)) {
        const val = manifest[key];
        const matchExample = val.match(/{{(.*)}}/);

        if ( ! isEmpty(matchExample)) {
          Object.assign(newJson, {
            [key]: val,
          });

          const folder = matchExample[1];
          parseTranslation(json[key], `${rootFolder}/${folder}`);
          json = omit(json, key);   // deletes the folder key so that we are
                                    // left with only with simple keys
        }

        else {
          if (json[key]) {
            Object.assign(newJson, {
              [key]: json[key],
            });

            json = omit(json, key);
          }
        }
      }
    }

    // now we are left only with simple keys, so add them to the manifest (newJson)
    Object.assign(newJson, json);
    fs.writeFile(`${rootFolder}/manifest.json`, JSON.stringify(newJson, null, 2), (err) => {
      if (err) throw err;
    });
{'fraction_non_alphanumeric': 0.13361236316806183, 'fraction_numerical': 0.00128783000643915, 'mean_word_length': 2.100798403193613, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6093292', 'n_tokens_mistral': 907, 'n_tokens_neox': 871, 'n_words': 257}