<commit_before>'use strict';

var EntitySchema = require('./EntitySchema'),
    ArraySchema = require('./ArraySchema'),
    isObject = require('lodash/lang/isObject'),
    isEqual = require('lodash/lang/isEqual');

function visitObject(obj, schema, bag, assignEntity) {
  var normalized = {};

  for (var prop in obj) {
    if (obj.hasOwnProperty(prop)) {
      var entity = visit(obj[prop], schema[prop], bag);
      if (assignEntity)
        assignEntity(normalized, prop, entity);
      else
        normalized[prop] = entity;
    }
  }

  return normalized;
}

function visitArray(obj, arraySchema, bag) {
  var itemSchema = arraySchema.getItemSchema(),
      normalized;

  normalized = obj.map(function (childObj) {
    return visit(childObj, itemSchema, bag);
  });

  return normalized;
}


function mergeIntoEntity(entityA, entityB, entityKey) {
  for (var prop in entityB) {
    if (!entityB.hasOwnProperty(prop)) {
      continue;
    }

    if (!entityA.hasOwnProperty(prop) || isEqual(entityA[prop], entityB[prop])) {
      entityA[prop] = entityB[prop];
      continue;
    }

    console.warn(
      'When merging two ' + entityKey + ', found shallowly unequal data in their "' + prop + '" values. Using the earlier value.',
      entityA[prop], entityB[prop]
    );
  }
}

function visitEntity(entity, entitySchema, bag, assignEntity) {
  var entityKey = entitySchema.getKey(),
      idAttribute = entitySchema.getIdAttribute(),
      id = entity[idAttribute],
      stored,
      normalized;

  if (!bag[entityKey]) {
    bag[entityKey] = {};
  }

  if (!bag[entityKey][id]) {
    bag[entityKey][id] = {};
  }

  stored = bag[entityKey][id];
  normalized = visitObject(entity, entitySchema, bag, assignEntity);

  mergeIntoEntity(stored, normalized, entityKey);

  return id;
}

function visit(obj, schema, bag, assignEntity) {
  if (!isObject(obj) || !isObject(schema)) {
    return obj;
  }

  if (schema instanceof EntitySchema) {
    return visitEntity(obj, schema, bag, assignEntity);
  } else if (schema instanceof ArraySchema) {
    return visitArray(obj, schema, bag);
  } else {
    return visitObject(obj, schema, bag, assignEntity);
  }
}

function normalize(obj, schema, assignEntity) {
  if (!isObject(obj) && !Array.isArray(obj)) {
    throw new Error('Normalize accepts an object or an array as its input.');
  }

  if (!isObject(schema) || Array.isArray(schema)) {
    throw new Error('Normalize accepts an object for schema.');
  }

  var bag = {},
      result = visit(obj, schema, bag, assignEntity);

  return {
    entities: bag,
    result: result
  };
}

function arrayOf(schema) {
  return new ArraySchema(schema);
}

module.exports = {
  Schema: EntitySchema,
  arrayOf: arrayOf,
  normalize: normalize
};
<commit_msg>pass assignEntity to all visit calls<commit_after>'use strict';

var EntitySchema = require('./EntitySchema'),
    ArraySchema = require('./ArraySchema'),
    isObject = require('lodash/lang/isObject'),
    isEqual = require('lodash/lang/isEqual');

function visitObject(obj, schema, bag, assignEntity) {
  var normalized = {};

  for (var prop in obj) {
    if (obj.hasOwnProperty(prop)) {
      var entity = visit(obj[prop], schema[prop], bag, assignEntity);
      if (assignEntity)
        assignEntity(normalized, prop, entity);
      else
        normalized[prop] = entity;
    }
  }

  return normalized;
}

function visitArray(obj, arraySchema, bag, assignEntity) {
  var itemSchema = arraySchema.getItemSchema(),
      normalized;

  normalized = obj.map(function (childObj) {
    return visit(childObj, itemSchema, bag, assignEntity);
  });

  return normalized;
}


function mergeIntoEntity(entityA, entityB, entityKey) {
  for (var prop in entityB) {
    if (!entityB.hasOwnProperty(prop)) {
      continue;
    }

    if (!entityA.hasOwnProperty(prop) || isEqual(entityA[prop], entityB[prop])) {
      entityA[prop] = entityB[prop];
      continue;
    }

    console.warn(
      'When merging two ' + entityKey + ', found shallowly unequal data in their "' + prop + '" values. Using the earlier value.',
      entityA[prop], entityB[prop]
    );
  }
}

function visitEntity(entity, entitySchema, bag, assignEntity) {
  var entityKey = entitySchema.getKey(),
      idAttribute = entitySchema.getIdAttribute(),
      id = entity[idAttribute],
      stored,
      normalized;

  if (!bag[entityKey]) {
    bag[entityKey] = {};
  }

  if (!bag[entityKey][id]) {
    bag[entityKey][id] = {};
  }

  stored = bag[entityKey][id];
  normalized = visitObject(entity, entitySchema, bag, assignEntity);

  mergeIntoEntity(stored, normalized, entityKey);

  return id;
}

function visit(obj, schema, bag, assignEntity) {
  if (!isObject(obj) || !isObject(schema)) {
    return obj;
  }

  if (schema instanceof EntitySchema) {
    return visitEntity(obj, schema, bag, assignEntity);
  } else if (schema instanceof ArraySchema) {
    return visitArray(obj, schema, bag, assignEntity);
  } else {
    return visitObject(obj, schema, bag, assignEntity);
  }
}

function normalize(obj, schema, assignEntity) {
  if (!isObject(obj) && !Array.isArray(obj)) {
    throw new Error('Normalize accepts an object or an array as its input.');
  }

  if (!isObject(schema) || Array.isArray(schema)) {
    throw new Error('Normalize accepts an object for schema.');
  }

  var bag = {},
      result = visit(obj, schema, bag, assignEntity);

  return {
    entities: bag,
    result: result
  };
}

function arrayOf(schema) {
  return new ArraySchema(schema);
}

module.exports = {
  Schema: EntitySchema,
  arrayOf: arrayOf,
  normalize: normalize
};

{'fraction_non_alphanumeric': 0.1339524915163422, 'fraction_numerical': 0.0, 'mean_word_length': 3.5826513911620292, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7267229', 'n_tokens_mistral': 1725, 'n_tokens_neox': 1716, 'n_words': 439}