<commit_before>import makeDebug from 'debug';
import { cloneDeep, compact, defaults, find, flatten, get, set, map } from 'lodash';
import fp from 'mostly-func';
import { plural } from 'pluralize';
import validator from 'validator';

const debug = makeDebug('mostly:feathers-mongoose:helpers');

// get field by path, supporting `array.field`
export function getField(item, field) {
  let parts = field.split('.');
  let value = item, part;
  while (parts.length) {
    part = parts.shift();
    if (Array.isArray(value)) {
      value = fp.pipe(
        fp.map(fp.prop(part)),
        fp.reject(fp.isNil),
        fp.flatten
      )(value);
    } else {
      value = value? value[part] : null;
    }
  }

  return Array.isArray(value) ? fp.filter(fp.identity, value) : value;
}

export function setFieldByKey(item, field, data) {
  let parts = field.split('.');
  let key = parts.pop();
  let value = item, part;
  while (parts.length) {
    part = parts.shift();
    value = value[part];
  }
  if (!value) return; // nothing to setField

  if (Array.isArray(value) && !value[key]) {
    value.forEach(function(v) {
      let id = v[key];
      if (Array.isArray(id)) {
        set(v, key, map(id, it => data[it] || it));
      } else {
        set(v, key, data[id] || id);
      }
    });
  } else {
    if (value && value[key]) {
      let id = value[key];
      if (Array.isArray(id)) {
        set(item, field, map(id, it => data[it] || it));
      } else {
        set(item, field, data[id] || id);
      }
    }
  }

  return cloneDeep(item);
}

// get field by path
export function setField(item, target, data, field, options) {
  //debug('setField ==== %s, \n  >>>> %s / %s, \n  >>>> %s ',
  //  util.inspect(item), target, field, util.inspect(data));
  options = options || {};

  field = field || target;
  let parts = field.split('.');
  let key = parts.pop();
  let value = item, part;
  while (parts.length) {
    part = parts.shift();
    if (value[part] !== undefined) value = value[part];
    if (fp.is(Array, value) && parts.length > 0) {
      let subValue = fp.map(fp.prop(parts[0]), value);
      if (fp.any(fp.is(Array), subValue)) {
        value = [].concat.apply([], subValue); // flatten the array of array
        continue;
      } else {
        key = [...parts, key].join('.');
        break;
      }
    }
  }

  if (!value) {
    warn(0, '<nothing-to-setField>');
    return; // nothing to setField
  }

  function warn(which, id) {
    debug(' >>> WARN %d: setField %s/%s(id=%j) not found with item %s',
      which, target, field, id, item[options.idField]);
  }

  // avoid duplicate setField
  function cloneById(v, k) {
    if (Array.isArray(v)) {
      let match = find(v, it => String(it[options.idField]) === String(k));
      return match ? cloneDeep(match) : (options.preserveOrigin ? k : null);
    } else {
      return String(v[options.idField]) === String(k)? cloneDeep(v) : (options.preserveOrigin ? k : null);
    }
  }

  if (Array.isArray(value)) {
    value.forEach(function(v) {
      let entry = get(v, key);
      if(Array.isArray(entry)) {
        set(v, key, compact(entry.map(it => {
          let id = options.path? it[options.idField] : it;
          let clone = cloneById(data, id);
          return options.retained? defaults(it, clone) : clone;
        })));
        if (!get(v, key)) warn(1, entry);
      } else {
        let id = options.path? entry[options.idField] : entry;
        let clone = cloneById(data, id);
        set(v, key, options.retained? defaults(entry, clone) : clone);
        if (!get(v, key)) warn(2, entry);
      }
    });
  } else {
    if (value && value[key]) {
      let entry = get(value, key);
      if (Array.isArray(entry)) {
        set(item, target, compact(entry.map(it => {
          let id = options.path? it[options.idField] : it;
          let clone = cloneById(data, id);
          return options.retained? defaults(it, clone) : clone;
        })));
        if (!get(item, target)) warn(3, entry);
      } else {
        let id = options.path? entry[options.idField] : entry;
        let clone = cloneById(data, id);
        set(item, target, options.retained? defaults(entry, clone) : clone);
        if (!get(item, target)) warn(4, entry);
      }
    } else {
      warn(4, '<no-such-field>');
    }
  }
  return item;
}

export function reorderPosition(Model, item, newPos, options = {}) {
  const prevPos = parseInt(item.position || 0);
  newPos = parseInt(newPos || 0);

  const whichWay = (newPos > prevPos) ? -1 : 1;
  const start = (newPos > prevPos) ? prevPos + 1 : newPos;
  const end = (newPos > prevPos) ? newPos : prevPos - 1;

  const cond = {
    position: { '$gte': start, '$lte': end }
  };<commit_msg>skip warning for populate path not exists<commit_after>import makeDebug from 'debug';
import { cloneDeep, compact, defaults, find, flatten, get, set, map } from 'lodash';
import fp from 'mostly-func';
import { plural } from 'pluralize';
import validator from 'validator';

const debug = makeDebug('mostly:feathers-mongoose:helpers');

// get field by path, supporting `array.field`
export function getField(item, field) {
  let parts = field.split('.');
  let value = item, part;
  while (parts.length) {
    part = parts.shift();
    if (Array.isArray(value)) {
      value = fp.pipe(
        fp.map(fp.prop(part)),
        fp.reject(fp.isNil),
        fp.flatten
      )(value);
    } else {
      value = value? value[part] : null;
    }
  }

  return Array.isArray(value) ? fp.filter(fp.identity, value) : value;
}

export function setFieldByKey(item, field, data) {
  let parts = field.split('.');
  let key = parts.pop();
  let value = item, part;
  while (parts.length) {
    part = parts.shift();
    value = value[part];
  }
  if (!value) return; // nothing to setField

  if (Array.isArray(value) && !value[key]) {
    value.forEach(function(v) {
      let id = v[key];
      if (Array.isArray(id)) {
        set(v, key, map(id, it => data[it] || it));
      } else {
        set(v, key, data[id] || id);
      }
    });
  } else {
    if (value && value[key]) {
      let id = value[key];
      if (Array.isArray(id)) {
        set(item, field, map(id, it => data[it] || it));
      } else {
        set(item, field, data[id] || id);
      }
    }
  }

  return cloneDeep(item);
}

// get field by path
export function setField(item, target, data, field, options) {
  //debug('setField ==== %s, \n  >>>> %s / %s, \n  >>>> %s ',
  //  util.inspect(item), target, field, util.inspect(data));
  options = options || {};

  field = field || target;
  let parts = field.split('.');
  let key = parts.pop();
  let value = item, part;
  while (parts.length) {
    part = parts.shift();
    if (value[part] !== undefined) value = value[part];
    if (fp.is(Array, value) && parts.length > 0) {
      let subValue = fp.map(fp.prop(parts[0]), value);
      if (fp.any(fp.is(Array), subValue)) {
        value = [].concat.apply([], subValue); // flatten the array of array
        continue;
      } else {
        key = [...parts, key].join('.');
        break;
      }
    }
  }

  if (!value) {
    warn(0, '<nothing-to-setField>');
    return; // nothing to setField
  }

  function warn(which, id) {
    debug(' >>> WARN %d: setField %s/%s(id=%j) not found with item %s',
      which, target, field, id, item[options.idField]);
  }

  // avoid duplicate setField
  function cloneById(v, k) {
    if (Array.isArray(v)) {
      let match = find(v, it => String(it[options.idField]) === String(k));
      return match ? cloneDeep(match) : (options.preserveOrigin ? k : null);
    } else {
      return String(v[options.idField]) === String(k)? cloneDeep(v) : (options.preserveOrigin ? k : null);
    }
  }

  if (Array.isArray(value)) {
    value.forEach(function(v) {
      let entry = get(v, key);
      if(Array.isArray(entry)) {
        set(v, key, compact(entry.map(it => {
          let id = options.path? it[options.idField] : it;
          let clone = cloneById(data, id);
          return options.retained? defaults(it, clone) : clone;
        })));
        if (!get(v, key)) warn(1, entry);
      } else {
        let id = options.path? entry[options.idField] : entry;
        let clone = cloneById(data, id);
        set(v, key, options.retained? defaults(entry, clone) : clone);
        if (!get(v, key)) warn(2, entry);
      }
    });
  } else {
    if (value && value[key]) {
      let entry = get(value, key);
      if (Array.isArray(entry)) {
        set(item, target, compact(entry.map(it => {
          let id = options.path? it[options.idField] : it;
          let clone = cloneById(data, id);
          return options.retained? defaults(it, clone) : clone;
        })));
        if (!get(item, target)) warn(3, entry);
      } else {
        let id = options.path? entry[options.idField] : entry;
        let clone = cloneById(data, id);
        set(item, target, options.retained? defaults(entry, clone) : clone);
        if (!get(item, target)) warn(4, entry);
      }
    } else {
      //warn(4, 'skip <no-such-field>');
    }
  }
  return item;
}

export function reorderPosition(Model, item, newPos, options = {}) {
  const prevPos = parseInt(item.position || 0);
  newPos = parseInt(newPos || 0);

  const whichWay = (newPos > prevPos) ? -1 : 1;
  const start = (newPos > prevPos) ? prevPos + 1 : newPos;
  const end = (newPos > prevPos) ? newPos : prevPos - 1;

  const cond = {
    position: { '$gte': start, '$lte': end }
  };
{'fraction_non_alphanumeric': 0.17361037865204093, 'fraction_numerical': 0.0029532749709946207, 'mean_word_length': 2.5566391597899476, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 47, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1403425', 'n_tokens_mistral': 3211, 'n_tokens_neox': 3199, 'n_words': 833}