<commit_before>const fs = require('fs');
const flat = require('flat');

function fixParagraphStructure(originalJSONPath, translatedJSONPath) {
  const originalJSONFile = fs.readFileSync(originalJSONPath);
  const originalJSON = flat.flatten(JSON.parse(originalJSONFile));
  const translJSONFile = fs.readFileSync(translatedJSONPath);
  const translJSON = flat.flatten(JSON.parse(translJSONFile));
  const newTranslatedObj = {};

  for (var key in originalJSON) {
    if (key in translJSON) {
      newTranslatedObj[key] = translJSON[key];
    } else {
      let root = key.slice(0, key.length - 2);
      if (root.endsWith('description') && root + '.0' in translJSON) {
        newTranslatedObj[key] = '';
      } else {
        newTranslatedObj[key] = originalJSON[key];
      }
    }
  }

  fs.writeFileSync(
    translatedJSONPath,
    JSON.stringify(flat.unflatten(newTranslatedObj), undefined, 2)
  );
}<commit_msg>Change delimiter in flatten function<commit_after>const fs = require('fs');
const flat = require('flat');

function fixParagraphStructure(originalJSONPath, translatedJSONPath) {
  const originalJSONFile = fs.readFileSync(originalJSONPath);
  const originalJSON = flat.flatten(JSON.parse(originalJSONFile), {
    delimiter: '/'
  });
  const translJSONFile = fs.readFileSync(translatedJSONPath);
  const translJSON = flat.flatten(JSON.parse(translJSONFile), {
    delimiter: '/'
  });
  const newTranslatedObj = {};

  for (var key in originalJSON) {
    if (key in translJSON) {
      newTranslatedObj[key] = translJSON[key];
    } else {
      let root = key.slice(0, key.length - 2);
      if (root.endsWith('description') && root + '/0' in translJSON) {
        newTranslatedObj[key] = '';
      } else {
        newTranslatedObj[key] = originalJSON[key];
      }
    }
  }

  fs.writeFileSync(
    translatedJSONPath,
    JSON.stringify(
      flat.unflatten(newTranslatedObj, { delimiter: '/' }),
      undefined,
      2
    )
  );
}
{'fraction_non_alphanumeric': 0.12914752424706483, 'fraction_numerical': 0.00408371618172537, 'mean_word_length': 3.6226415094339623, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6114978', 'n_tokens_mistral': 625, 'n_tokens_neox': 605, 'n_words': 118}