<commit_before>var fs = require('fs'),
    path = require('path'),
    parser = require('./stream.js'),
    persister = require('./adapter.js');

var filenames = {
  child: {
    localdir: path.join(__dirname, '..', '..', 'static', 'db', 'bmi-child'),
    files: [
      ['length_height', ['lhfa_boys_p_exp', 'lhfa_girls_p_exp']],
      ['weight', ['wfa_boys_p_exp', 'wfa_girls_p_exp']],
      ['bmi', ['bfa_boys_p_exp', 'bfa_girls_p_exp']],
      ['weight_for_length', ['wfl_boys_p_exp', 'wfl_girls_p_exp']],
      ['weigth_for_height', ['wfh_boys_p_exp', 'wfh_girls_p_exp']],
      ['arm_circunference', ['acfa_boys_p_exp', 'acfa_girls_p_exp']],
      ['subscapular_skinfold', ['ssfa_boys_p_exp', 'ssfa_girls_p_exp']],
      ['triceps_skinfold', ['tsfa_boys_p_exp', 'tsfa_girls_p_exp']]
    ]
  },
  young: {
    localdir: path.join(__dirname, '..', '..', 'static', 'db', 'bmi-young'),
    files: [
      ['bmi', ['bmi_boys_perc_WHO2007_exp', 'bmi_girls_perc_WHO2007_exp']],
      ['weight', ['wfa_boys_perc_WHO2007_exp', 'wfa_girls_perc_WHO2007_exp']],
      ['height', ['hfa_boys_perc_WHO2007_exp', 'hfa_girls_perc_WHO2007_exp']]
    ]
  }
};

Object.keys(filenames).forEach(function (key) {
  var info = filenames[key];
  info.files.forEach(function (files) {
    files[1].forEach(function (basename) {
      var dbfile = fs.createReadStream(path.join(info.localdir, basename + '.txt')),
          lmsinfo = parser({delimiter: '\t', newline: '\n'}),
          inmemory = persister();

      inmemory.on('finish', function () {
        console.log(
          basename,
          Object.keys(inmemory.memory.xcoord).length,
          Math.min.apply(null, (Object.keys(inmemory.memory.xcoord))),
          Math.max.apply(null, (Object.keys(inmemory.memory.xcoord)))
        );
      });

      dbfile.pipe(lmsinfo).pipe(inmemory);
    });
  });
});<commit_msg>Read the csv WHO data and convert into a proper json format.<commit_after>var fs = require('fs'),
    path = require('path'),
    parser = require('./stream.js'),
    converter = require('./adapter.js');

var filenames = {
  child: {
    localdir: path.join(__dirname, 'db', 'bmi-child'),
    todir: path.join(__dirname, 'db'),
    files: [
      {
        from: 'lhfa_boys_p_exp',
        to: 'length-child_height_for_age_male'
      },
      {
        from: 'lhfa_girls_p_exp',
        to: 'length-child_height_for_age_female'
      },
      {
        from: 'wfa_boys_p_exp',
        to: 'child_weight_for_age_male'
      },
      {
        from: 'wfa_girls_p_exp',
        to: 'child_weight_for_age_female'
      },
      {
        from: 'bfa_boys_p_exp',
        to: 'child_bmi_for_age_male'
      },
      {
        from: 'bfa_girls_p_exp',
        to: 'child_bmi_for_age_female'
      },
      {
        from: 'wfl_boys_p_exp',
        to: 'child_weight_for_length_male'
      },
      {
        from: 'wfl_girls_p_exp',
        to: 'child_weight_for_length_female'
      },
      {
        from: 'wfh_boys_p_exp',
        to: 'child_weight_for_height_male'
      },
      {
        from: 'wfh_girls_p_exp',
        to: 'child_weight_for_height_female'
      },
      {
        from: 'acfa_boys_p_exp',
        to: 'child_arm_circ_for_age_male'
      },
      {
        from: 'acfa_girls_p_exp',
        to: 'child_arm_circ_for_age_female'
      },
      {
        from: 'ssfa_boys_p_exp',
        to: 'child_subscapular_skinfold_for_age_male'
      },
      {
        from: 'ssfa_girls_p_exp',
        to: 'child_subscapular_skinfold_for_age_female'
      },
      {
        from: 'tsfa_boys_p_exp',
        to: 'child_triceps_skinfod_for_age_male'
      },
      {
        from: 'tsfa_girls_p_exp',
        to: 'child_triceps_skinfod_for_age_female'
      }
    ]
  },
  young: {
    localdir: path.join(__dirname, 'db', 'bmi-young'),
    todir: path.join(__dirname, 'db'),
    files: [
      {
        from: 'bmi_boys_perc_WHO2007_exp',
        to: 'young_bmi_for_age_male'
      },
      {
        from: 'bmi_girls_perc_WHO2007_exp',
        to: 'young_bmi_for_age_female'
      },
      {
        from: 'wfa_boys_perc_WHO2007_exp',
        to: 'young_weight_for_age_male'
      },
      {
        from: 'wfa_girls_perc_WHO2007_exp',
        to: 'young_weight_for_age_female'
      },
      {
        from: 'hfa_boys_perc_WHO2007_exp',
        to: 'young_height_for_age_male'
      },
      {
        from: 'hfa_girls_perc_WHO2007_exp',
        to: 'young_height_for_age_female'
      }
    ]
  }
};

Object.keys(filenames).forEach(function (key) {
  var info = filenames[key];
  info.files.forEach(function (files) {
    var dbfile = fs.createReadStream(path.join(info.localdir, files.from + '.txt')),
        lmsparser = parser({delimiter: '\t', newline: '\n'}),
        lmsconverter = converter(),
        dbjson = fs.createWriteStream(path.join(info.todir, files.to + '.json'));

    dbfile.pipe(lmsparser).pipe(lmsconverter).pipe(dbjson);
  });
});
{'fraction_non_alphanumeric': 0.1537055837563452, 'fraction_numerical': 0.009949238578680204, 'mean_word_length': 2.616740088105727, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '714109', 'n_tokens_mistral': 1986, 'n_tokens_neox': 1891, 'n_words': 241}