<commit_before>var _ = require('lodash');

var currentLocation = [];
var obj = {};

var levelRegex = /^"([A-Za-z0-9\_\-]*)"$/;
var propertyRegex = /^"([A-Za-z1-9_\-]*)"[\t\ ]+"(.*)"$/;

function convertToArray(location) {
  var currentValue = _.get(obj, location);
  _.set(obj, location, [currentValue]);
}

function stripComments(line) {
  return line.replace(/\/\/.+/, '');
}

function parseLine(line) {

  line = line.trim();

  line = stripComments(line);
  var match;

  if (match = line.match(levelRegex)) {
    var matchedKey = match[1];
    var nextLocation = _.chain(currentLocation).clone().push(matchedKey).value().join('.');

    // If this key already exists, convert it to an array or append to the array
    var checkKey = _.get(obj, nextLocation);
    if (checkKey && !_.isArray(checkKey)) {
      convertToArray(nextLocation);
      currentLocation.push(matchedKey + '[1]');
    } else if (checkKey){
      nextLocationLength = _.get(obj, nextLocation).length;
      currentLocation.push(matchedKey + '[' + nextLocationLength + ']');
    } else {
      currentLocation.push(match[1]);
    }
  } else if (match = line.match(propertyRegex)) {
    _.set(obj, currentLocation.join('.') + '.' + match[1], match[2]);
  }

  if (line === '}') {
    currentLocation.pop();
  }

}

function parse(vdf) {
  obj = {};

  var lines = vdf.split('\n');

  _.forEach(lines, function(line) {
    parseLine(line);<commit_msg>Parse commented lines and arrays<commit_after>var _ = require('lodash');

var currentLocation = [];
var obj = {};

var levelRegex = /^"([A-Za-z0-9\_\-]*)"$/;
var propertyRegex = /^"([A-Za-z1-9_\-]*)"[\t\ ]+"(.*)"$/;

function convertToArray(location) {
  var currentValue = _.get(obj, location);
  _.set(obj, location, [currentValue]);
}

function stripComments(line) {
  return line.replace(/\/\/.+/, '');
}

function parseLine(line) {

  line = stripComments(line);
  line = line.trim();
  var match;

  if (match = line.match(levelRegex)) {
    var matchedKey = match[1];
    var nextLocation = _.chain(currentLocation).clone().push(matchedKey).value().join('.');

    // If this key already exists, convert it to an array or append to the array
    var checkKey = _.get(obj, nextLocation);
    if (checkKey && !_.isArray(checkKey)) {
      convertToArray(nextLocation);
      currentLocation.push(matchedKey + '[1]');
    } else if (checkKey) {
      nextLocationLength = _.get(obj, nextLocation).length;
      currentLocation.push(matchedKey + '[' + nextLocationLength + ']');
    } else {
      currentLocation.push(match[1]);
    }
  } else if (match = line.match(propertyRegex)) {

    // If this key already exists, append to the array
    var nextLocation = currentLocation.join('.') + '.' + match[1];
    var checkKey = _.get(obj, nextLocation);
    if (checkKey) {
      if (!_.isArray(checkKey)) {
        convertToArray(nextLocation);
        console.log(nextLocation);
        checkKey = _.get(obj, nextLocation);
        checkKey.push(match[2]);
      } else {
        checkKey.push(match[2]);
      }
    } else {
      _.set(obj, currentLocation.join('.') + '.' + match[1], match[2]);
    }
  }

  if (line === '}') {
    currentLocation.pop();
  }

}

function parse(vdf) {
  obj = {};

  var lines = vdf.split('\n');

  _.forEach(lines, function(line) {
    parseLine(line);
{'fraction_non_alphanumeric': 0.18710454956312142, 'fraction_numerical': 0.006327206990057246, 'mean_word_length': 3.223918575063613, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1573046', 'n_tokens_mistral': 1121, 'n_tokens_neox': 1100, 'n_words': 227}