<commit_before>function Entity(name) {
  this.name = name;
  this.fields = [];
}

Entity.prototype.add = function(fieldInfo) {
  this.fields.push(fieldInfo);
};

var ModelBuilder = {
  walk: function(ast) {
    this.entities = [];
    this.name = 'model'
    if(ast.type === 'Program') {
      this.parseProgramBody(ast.body);
      return {
        name: this.name,
        entities: this.entities
      }
    }
  }
};

function _throw(msg) {
  throw new Error(msg);
};

ModelBuilder.parseProgramBody = function parseProgramBody(statements) {
  var self = this;
  statements.forEach(function(a){
    if (a.type === 'VariableDeclaration') {
      self.parseVariableDeclaration(a);
    }
    else {
      throw new Error('Unknown statement: ' + a.type);
    }
  });
}

ModelBuilder.parseVariableDeclaration = function parseVariableDeclaration(ast) {
  var self = this;

  console.assert(ast.type === 'VariableDeclaration');
  ast.declarations.forEach(function(dec) {
    // var entity = self.parseVariableDeclarator(dec);
    // // console.log(JSON.stringify(entity));
    // self.entities.push(entity);
    self.parseVariableDeclarator(dec);
  });
};

ModelBuilder.parseVariableDeclarator = function parseVariableDeclarator(ast) {
  var self = this;
  console.assert(ast.type === 'VariableDeclarator');

  if (ast.init) {
    if (ast.init.type === 'ObjectExpression') {
      var entity = new Entity(ast.id.name);
      var fields = this.parseObjectExpression(ast.init);
      entity.fields = fields;
      self.entities.push(entity);
    }
    else if (ast.init.type === 'ArrayExpression') {
      self.name = ast.id.name;
    }
    else {
      throw new Error('VariableDeclarator parsing: Unknown init type: ' + ast.init.type);
    }
  }
  else {
    throw new Error('VariableDeclarator has no init')
  }
}

ModelBuilder.parseObjectExpression = function parseObjectExpression(ast, firstLevel) {
  var self = this;

  if (typeof firstLevel === 'undefined') {
    firstLevel = true;
  }
  console.assert(ast.type === 'ObjectExpression');

  if (firstLevel) {
    var fields = [];
    ast.properties.forEach(function(prop){
      if (prop.type === 'Property') {
        fields.push({
          type: 'field',
          name: prop.key.name,
          def: self.parseDef(prop.value)
        })
      }
      else {
        _throw('ObjectExpression: Unknown property type: ' + prop.type);
      }
    });
    return fields;
  }
  else {
    var obj = {};
    ast.properties.forEach(function(prop) {
      obj[prop.key.name] = self.parsePropertyValue(prop.value);
    });
    return obj;
  }

}

ModelBuilder.parseDef = function parseDef(ast) {
  var self = this;
  if (ast.type === 'ObjectExpression') {
    var def = self.parseObjectExpression(ast, false);
    return def;
  }
  else {
    _throw('parseDef: Unknown type: ' + ast.type);
  }
};

ModelBuilder.parsePropertyValue = function parsePropertyValue(ast) {
  var self = this;
  if (ast.type === 'ObjectExpression') {
    return self.parseObjectExpression(ast, false);
  }
  else if(ast.type === 'Literal') {
    return ast.value;
  }
  else {
    _throw('parsePropertyValue: Unknown type: ' + ast.type);
  }
}
<commit_msg>failing test<commit_after>function Entity(name) {
  this.name = name;
  this.fields = [];
}

Entity.prototype.add = function(fieldInfo) {
  this.fields.push(fieldInfo);
};

var ModelBuilder = {
  walk: function(ast) {
    this.entities = [];
    this.name = 'model'
    if(ast.type === 'Program') {
      this.parseProgramBody(ast.body);
      return {
        name: this.name,
        entities: this.entities
      }
    }
  }
};

function _throw(msg) {
  throw new Error(msg);
};

ModelBuilder.parseProgramBody = function parseProgramBody(statements) {
  var self = this;
  statements.forEach(function(a){
    if (a.type === 'VariableDeclaration') {
      self.parseVariableDeclaration(a);
    }
    else {
      throw new Error('Unknown statement: ' + a.type);
    }
  });
}

ModelBuilder.parseVariableDeclaration = function parseVariableDeclaration(ast) {
  var self = this;

  console.assert(ast.type === 'VariableDeclaration');
  ast.declarations.forEach(function(dec) {
    // var entity = self.parseVariableDeclarator(dec);
    // // console.log(JSON.stringify(entity));
    // self.entities.push(entity);
    self.parseVariableDeclarator(dec);
  });
};

ModelBuilder.parseVariableDeclarator = function parseVariableDeclarator(ast) {
  var self = this;
  console.assert(ast.type === 'VariableDeclarator');

  if (ast.init) {
    if (ast.init.type === 'ObjectExpression') {
      var entity = new Entity(ast.id.name);
      var fields = this.parseObjectExpression(ast.init);
      entity.fields = fields;
      self.entities.push(entity);
    }
    // else if (ast.init.type === 'ArrayExpression') {
    //   self.name = ast.id.name;
    // }
    else {
      throw new Error('VariableDeclarator parsing: Unknown init type: ' + ast.init.type);
    }
  }
  else {
    throw new Error('VariableDeclarator has no init')
  }
};

ModelBuilder.parseObjectExpression = function parseObjectExpression(ast, firstLevel) {
  var self = this;

  if (typeof firstLevel === 'undefined') {
    firstLevel = true;
  }
  console.assert(ast.type === 'ObjectExpression');

  if (firstLevel) {
    var fields = [];
    ast.properties.forEach(function(prop){
      if (prop.type === 'Property') {
        fields.push({
          type: 'field',
          name: prop.key.name,
          def: self.parseDef(prop.value)
        })
      }
      else {
        _throw('ObjectExpression: Unknown property type: ' + prop.type);
      }
    });
    return fields;
  }
  else {
    var obj = {};
    ast.properties.forEach(function(prop) {
      obj[prop.key.name] = self.parsePropertyValue(prop.value);
    });
    return obj;
  }

}

ModelBuilder.parseDef = function parseDef(ast) {
  var self = this;
  if (ast.type === 'ObjectExpression') {
    var def = self.parseObjectExpression(ast, false);
    return def;
  }
  else {
    _throw('parseDef: Unknown type: ' + ast.type);
  }
};

ModelBuilder.parsePropertyValue = function parsePropertyValue(ast) {
  var self = this;
  if (ast.type === 'ObjectExpression') {
    return self.parseObjectExpression(ast, false);
  }
  else if(ast.type === 'Literal') {
    return ast.value;
  }
  else {
    _throw('parsePropertyValue: Unknown type: ' + ast.type);
  }
}

{'fraction_non_alphanumeric': 0.14474304573314475, 'fraction_numerical': 0.0, 'mean_word_length': 2.965109034267913, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7505967', 'n_tokens_mistral': 1997, 'n_tokens_neox': 2000, 'n_words': 387}