<commit_before>'use strict';
const flatten = require('flat');
const ObjectId = require('./object_id');
const pluralize = require('pluralize');
const NESTED_PATH_INDEX_REPLACER = /^(.+\.){1,}(.+)$/;
const IGNORE_LOKI_FIELDS = ['meta.revision', 'meta.created', 'meta.version', '$loki'];

var isExternalRef = function(key) {
  let properties = key.split('.');
  return (properties[properties.length - 1] === 'ref');
};

var isTypeDef = function(key) {
  let properties = key.split('.');
  return (properties[properties.length - 1] === 'type');
};

var isDefaultDef = function(key) {
  let properties = key.split('.');
  return (properties[properties.length - 1] === 'default');
};

/**
 * proxy for creating new loki documents
 * 
 * @class lowkieSchema
 */
class lowkieSchema {
  constructor(scheme, lowkieSingleton, db = 'default') {
    this.scheme = scheme;
    this.flattenedScheme = flatten(scheme);
    let flatKeys = Object.keys(this.flattenedScheme);
    this.default_defs = flatKeys.filter(isDefaultDef);
    this.external_refs = flatKeys.filter(isExternalRef);
    this.type_defs = flatKeys.filter(isTypeDef);
    this.validNames = Object.keys(this.flattenedScheme).reduce((result, key) => {
      let properties = key.split('.');
      let parentKey = properties.slice(0, properties.length - 1).join('.');
      if (this.default_defs.indexOf(key) === -1 && this.external_refs.indexOf(key) === -1 && this.type_defs.indexOf(key) === -1) {
        result.push(key);
      } else if (result.indexOf(parentKey) === -1) {
        result.push(parentKey);
      }
      return result;
    }, []).concat(['_id']);
    this.lowkie = lowkieSingleton;
    this.createDoc = this.createDocument.bind(this);
    this.dbconnection = db;
    if (!lowkieSingleton) {
      throw new Error('Invalid schema invocation, try using lowkie.Schema instead of Schema');
    }
    return this;
  }
  validateDocument(original_document) {
    let _document = Object.assign({}, flatten(original_document), original_document);
      _document = Object.keys(_document).reduce((result, key) => {
        if (_document[key]) {
          let indexedKey = key.replace(/^(.+\.){1,}(?:\d+\.)(.+)$/, '$10.$2');
          let validType = this.flattenedScheme[`${ key }.type`] || this.flattenedScheme[key] || this.flattenedScheme[`${ indexedKey }.type`] || this.flattenedScheme[indexedKey];
          if (validType === Date) {
            result[key] === new Date();
          }
          if (typeof _document[key] === 'string' && (validType === String || validType === ObjectId)) {
            result[key] = _document[key].toString();
          } else if (validType === Boolean) {
            result[key] = (_document[key]) ? true : false;
          } else if (validType === Number) {
            result[key] = Number(_document[key]);
          } else if (typeof _document[key] === 'object' && (validType === Array || validType === Object)) {
            result[key] = _document[key];
          }
        } else if (typeof original_document[ key ] === 'object' && typeof this.flattenedScheme[ key ] === 'function' && this.flattenedScheme[ key ] === Object) { //schema type mixed
          result[key] = original_document[key];
        }
        return result;
      }, { _id: _document._id });

      this.default_defs.forEach(key => {
        let documentKey = key.substring(0, key.indexOf('.default'));
        if (typeof this.flattenedScheme[key] === 'function') {
          if (this.flattenedScheme[`${ documentKey }.type`] === Date) {
            _document[documentKey] = (_document[documentKey]) ? _document[documentKey] : new Date();
          } else {
            _document[documentKey] = (_document[documentKey]) ? _document[documentKey] : this.flattenedScheme[key]();
          }
        } else {
          _document[documentKey] = (_document[documentKey]) ? _document[documentKey] : this.flattenedScheme[key];
        }
      });
      return flatten.unflatten(_document);
    }
    /**
     * returns validated document for lokijs
     * 
     * @param {any} doc 
     * @returns object
     * 
     * @memberOf lowkieSchema
     */
  createDocument(doc) {
      let newDoc = Object.assign({
        _id: ObjectId.createId(),
      }, doc);
      return this.validateDocument(newDoc);
    }
    /**
     * overwrites the default insert method
     * 
     * @param {any} options 
     * @returns Promise
     * 
     * @memberOf lowkieSchema
     */
  insert(options = {}) {
      let lokiCollectionInsert = options.target;
      let lowkieInstance = options.thisArg;
      let lowkieDocument = options.argumentsList;

      // let { target, thisArg, argumentsList, } = options;
      return new Promise((resolve, reject) => {
        try {
          let newDoc = (Array.isArray(lowkieDocument)) ?
            lowkieDocument.map(lowkiedoc => this.createDoc(lowkiedoc)) :
            this.createDoc(lowkieDocument);
          lokiCollectionInsert.call(lowkieInstance, newDoc);
          this.lowkie.dbs[this.dbconnection].saveDatabase((err) => {
            if (err) reject(err);
            else return resolve(newDoc);
          });
        } catch (e) {
          reject(e);
        }
      });<commit_msg>Adding non strict schema updates<commit_after>'use strict';
const flatten = require('flat');
const ObjectId = require('./object_id');
const pluralize = require('pluralize');
const NESTED_PATH_INDEX_REPLACER = /^(.+\.){1,}(.+)$/;
const IGNORE_LOKI_FIELDS = ['meta.revision', 'meta.created', 'meta.version', '$loki'];

var isExternalRef = function(key) {
  let properties = key.split('.');
  return (properties[properties.length - 1] === 'ref');
};

var isTypeDef = function(key) {
  let properties = key.split('.');
  return (properties[properties.length - 1] === 'type');
};

var isDefaultDef = function(key) {
  let properties = key.split('.');
  return (properties[properties.length - 1] === 'default');
};

/**
 * proxy for creating new loki documents
 * 
 * @class lowkieSchema
 */
class lowkieSchema {
  constructor(scheme, lowkieSingleton, db = 'default') {
    this.scheme = scheme;
    this.flattenedScheme = flatten(scheme);
    let flatKeys = Object.keys(this.flattenedScheme);
    this.default_defs = flatKeys.filter(isDefaultDef);
    this.external_refs = flatKeys.filter(isExternalRef);
    this.type_defs = flatKeys.filter(isTypeDef);
    this.validNames = Object.keys(this.flattenedScheme).reduce((result, key) => {
      let properties = key.split('.');
      let parentKey = properties.slice(0, properties.length - 1).join('.');
      if (this.default_defs.indexOf(key) === -1 && this.external_refs.indexOf(key) === -1 && this.type_defs.indexOf(key) === -1) {
        result.push(key);
      } else if (result.indexOf(parentKey) === -1) {
        result.push(parentKey);
      }
      return result;
    }, []).concat(['_id']);
    this.lowkie = lowkieSingleton;
    this.createDoc = this.createDocument.bind(this);
    this.dbconnection = db;
    if (!lowkieSingleton) {
      throw new Error('Invalid schema invocation, try using lowkie.Schema instead of Schema');
    }
    return this;
  }
  validateDocument(original_document) {
    let _document = Object.assign({}, flatten(original_document), original_document);
      _document = Object.keys(_document).reduce((result, key) => {
        if (_document[key]) {
          let indexedKey = key.replace(/^(.+\.){1,}(?:\d+\.)(.+)$/, '$10.$2');
          let validType = this.flattenedScheme[`${ key }.type`] || this.flattenedScheme[key] || this.flattenedScheme[`${ indexedKey }.type`] || this.flattenedScheme[indexedKey];
          if (validType === Date) {
            result[key] === new Date();
          }
          if (typeof _document[key] === 'string' && (validType === String || validType === ObjectId)) {
            result[key] = _document[key].toString();
          } else if (validType === Boolean) {
            result[key] = (_document[key]) ? true : false;
          } else if (validType === Number) {
            result[key] = Number(_document[key]);
          } else if (typeof _document[key] === 'object' && (validType === Array || validType === Object)) {
            result[key] = _document[key];
          }
        } else if (typeof original_document[ key ] === 'object' && typeof this.flattenedScheme[ key ] === 'function' && this.flattenedScheme[ key ] === Object) { //schema type mixed
          result[key] = original_document[key];
        }
        return result;
      }, { _id: _document._id });

      this.default_defs.forEach(key => {
        let documentKey = key.substring(0, key.indexOf('.default'));
        if (typeof this.flattenedScheme[key] === 'function') {
          if (this.flattenedScheme[`${ documentKey }.type`] === Date) {
            _document[documentKey] = (_document[documentKey]) ? _document[documentKey] : new Date();
          } else {
            _document[documentKey] = (_document[documentKey]) ? _document[documentKey] : this.flattenedScheme[key]();
          }
        } else {
          _document[documentKey] = (_document[documentKey]) ? _document[documentKey] : this.flattenedScheme[key];
        }
      });
      return flatten.unflatten(_document);
    }
    /**
     * returns validated document for lokijs
     * 
     * @param {any} doc 
     * @returns object
     * 
     * @memberOf lowkieSchema
     */
  createDocument(doc) {
      let newDoc = Object.assign({
        _id: ObjectId.createId(),
      }, doc);
      return this.validateDocument(newDoc);
    }
    /**
     * overwrites the default insert method
     * 
     * @param {any} options 
     * @returns Promise
     * 
     * @memberOf lowkieSchema
     */
  insert(options = {}) {
      let lokiCollectionInsert = options.target;
      let lowkieInstance = options.thisArg;
      let lowkieDocument = options.argumentsList;
    // console.log({this.lowkie})
    // console.log('this.lowkie',this.lowkie)

      // let { target, thisArg, argumentsList, } = options;
      return new Promise((resolve, reject) => {
        try {
          let newDoc = lowkieDocument;
          if (this.lowkie && this.lowkie.config && this.lowkie.config.strictSchemas) {
            newDoc = (Array.isArray(lowkieDocument))
            ? lowkieDocument.map(lowkiedoc => this.createDoc(lowkiedoc))
            : this.createDoc(lowkieDocument);
            lokiCollectionInsert.call(lowkieInstance, newDoc);
          } else {
            lokiCollectionInsert.call(lowkieInstance, lowkieDocument);
          }
          this.lowkie.dbs[this.dbconnection].saveDatabase((err) => {
            if (err) reject(err);
            else return resolve(newDoc);
          });
        } catch (e) {
          reject(e);
        }
      });
{'fraction_non_alphanumeric': 0.15737368076959, 'fraction_numerical': 0.002801905295601009, 'mean_word_length': 2.7717506164142303, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '564975', 'n_tokens_mistral': 3239, 'n_tokens_neox': 3195, 'n_words': 645}