<commit_before>'use strict';

var async = require('async');
var _ = require('lodash');
var debug = {
  getFormsWithUniqueComponents: require('debug')('formio:update:3.0.5-getFormsWithUniqueComponents'),
  fixEachForm: require('debug')('formio:update:3.0.5-fixEachForm'),
  fixFormUniques: require('debug')('formio:update:3.0.5-fixFormUniques')
};

/**
 * Update 3.0.5
 *
 * This update does the following.
 *
 *   1.) Finds all forms that have components with unique properties
 *   2.) Coerces all unique fields to be comparable
 *
 * @param db
 * @param config
 * @param tools
 * @param done
 */
module.exports = function(db, config, tools, done) {
  var formCollection = db.collection('forms');
  var submissionCollection = db.collection('submissions');
  var blackListedComponents = ['select', 'address'];

  // The list of fixed forms.
  var fixedForms = [];

  /**
   * Fix the submissions unique fields.
   *
   * @param submission
   * @param uniques
   * @param next
   * @returns {*}
   */
  var fixSubmissionUniques = function(submission, uniques, next) {
    var update = {};
    _.each(uniques, function(path, key) {
      var item = _.get(submission, 'data.' + path);
      if (item) {
        // Coerce all unique string fields to be lowercase.
        if (typeof item === 'string') {
          update['data.' + path] = item.toString().toLowerCase();
        }
        // Coerce all unique string fields in an array to be lowercase.
        else if (item instanceof Array && (item.length > 0) && (typeof item[0] === 'string')) {
          _.map(item, function(element) {
            return element.toString().toLowerCase();
          });

          // Coerce all unique string fields to be lowercase.
          update['data.' + path] = item;
        }
      }
    });

    if (Object.keys(update).length === 0) {
      return next();
    }

    submissionCollection.update(
      {_id: tools.util.idToBson(submission._id)},
      {$set: update},
      function(err) {
        if (err) {
          return next(err);
        }


        return next();
      }
    );
  };

  /**
   * Fix all the submissions of the given form, to have comparable unique fields.
   *
   * @param form
   * @param next
   */
  var fixFormUniques = function(form, next) {
    var uniques = {};

    debug.fixFormUniques(form._id.toString());
    async.waterfall([
      function buildUniqueComponentList(callback) {
        tools.util.eachComponent(form.components, function(component, path) {
          // We only care about non-layout components.
          if (!_.get(component, 'key')) {
            return;
          }

          // We only care about unique components, that have not been blacklisted.
          if (_.get(component, 'unique') !== true || blackListedComponents.indexOf(_.get(component, 'type')) !== -1) {
            return;
          }

          debug.fixFormUniques('buildUniqueComponentList');
          uniques[component.key] = path;
        }, true);

        return callback();
      },
      function getFormSubmissions(callback) {
        debug.fixFormUniques('getFormSubmissions');
        submissionCollection.find({form: tools.util.idToBson(form._id)})
          .snapshot(true)
          .toArray(function(err, submissions) {
            if (err) {
              return callback(err);
            }

            debug.fixFormUniques('Form: ' + form._id.toString() + ', Submissions: ' + submissions.length);
            return callback(null, submissions);
          });
      },
      function fixEachSubmission(submissions, callback) {
        if (!submissions || submissions.length === 0) {
          return callback();
        }

        debug.fixFormUniques(uniques);
        async.each(submissions, function(submission, cb) {
          fixSubmissionUniques(submission, uniques, cb);
        }, callback);
      }
    ],
    function(err) {
      if (err) {
        return next(err);
      }

      fixedForms.push(tools.util.idToBson(form._id));
      debug.fixFormUniques('Form Done: ' + form._id.toString());
      next();
    });
  };

  /**
   * Get all the forms with a unique component in its root components list.
   *
   * @param next
   */
  var getFormsWithUniqueComponents = function(next) {
    formCollection.find({
      components: {$elemMatch: {unique: true, type: {$nin: blackListedComponents}}},
      deleted: {$eq: null}
    })
    .snapshot(true)
    .toArray(function(err, forms) {
      if (err) {
        return next(err);
      }

      debug.getFormsWithUniqueComponents(forms.length);
      return next(null, forms);
    });
  };

  /**
   * Get all the forms with a unique component within a layout component, that hasnt already been modified.
   *
   * @param next
   */
  var getFormsWithUniqueComponentsInLayoutComponents = function(next) {
    formCollection.find({
      _id: {$nin: fixedForms},
      deleted: {$eq: null},
      components: {
        $elemMatch: {
          $or: [
            {$and: [{columns: {$exists: true}}, {columns: {$elemMatch: {unique: true, type: {$nin: blackListedComponents}}}}]},
            {$and: [{rows: {$exists: true}}, {rows: {$elemMatch: {unique: true, type: {$nin: blackListedComponents}}}}]},
            {$and: [{components: {$exists: true}}, {components: {$elemMatch: {unique: true, type: {$nin: blackListedComponents}}}}]}
          ]
        }
      }
    })
    .snapshot(true)
    .toArray(function(err, forms) {
      if (err) {
        return next(err);
      }

      return next(null, forms);
    });
  };

  /**
   * Get all the forms with a unique component within a layout component in a layout component, that hasnt already been modified.
   *
   * @param next
   */
  var getFormsWithUniqueComponentsInLayoutComponentsInLayoutComponents = function(next) {
    formCollection.find({
      _id: {$nin: fixedForms},
      deleted: {$eq: null},
      components: {
        $elemMatch: {
          $or: [
            {columns: {$elemMatch: {
              $or: [
                {$and: [{columns: {$exists: true}}, {columns: {$elemMatch: {unique: true, type: {$nin: blackListedComponents}}}}]},
                {$and: [{rows: {$exists: true}}, {rows: {$elemMatch: {unique: true, type: {$nin: blackListedComponents}}}}]},
                {$and: [{components: {$exists: true}}, {components: {$elemMatch: {unique: true, type: {$nin: blackListedComponents}}}}]}
              ]
            }}},
            {rows: {$elemMatch: {
              $or: [
                {$and: [{columns: {$exists: true}}, {columns: {$elemMatch: {unique: true, type: {$nin: blackListedComponents}}}}]},
                {$and: [{rows: {$exists: true}}, {rows: {$elemMatch: {unique: true, type: {$nin: blackListedComponents}}}}]},
                {$and: [{components: {$exists: true}}, {components: {$elemMatch: {unique: true, type: {$nin: blackListedComponents}}}}]}
              ]
            }}},
            {components: {$elemMatch: {
              $or: [
                {$and: [{columns: {$exists: true}}, {columns: {$elemMatch: {unique: true, type: {$nin: blackListedComponents}}}}]},
                {$and: [{rows: {$exists: true}}, {rows: {$elemMatch: {unique: true, type: {$nin: blackListedComponents}}}}]},
                {$and: [{components: {$exists: true}}, {components: {$elemMatch: {unique: true, type: {$nin: blackListedComponents}}}}]}
              ]
            }}}
          ]
        }
      }
    })
    .snapshot(true)
    .toArray(function(err, forms) {
      if (err) {
        return next(err);
      }

      return next(null, forms);
    });
  };

  /**
   * Get all the forms with a layout component in a layout component, that hasnt already been modified.
   *
   * @param next
   */
  var getFormsWithPotentialUniqueComponentsInLayoutComponents = function(next) {
    formCollection.find({
      _id: {$nin: fixedForms},
      deleted: {$eq: null},
      components: {
        $elemMatch: {
          $or: [
            {columns: {$elemMatch: {
              $or: [
                {columns: {$exists: true}},
                {rows: {$exists: true}},
                {components: {$exists: true}}
              ]
            }}},
            {rows: {$elemMatch: {
              $or: [
                {columns: {$exists: true}},
                {rows: {$exists: true}},
                {components: {$exists: true}}
              ]
            }}},
            {components: {$elemMatch: {
              $or: [
                {columns: {$exists: true}},
                {rows: {$exists: true}},
                {components: {$exists: true}}
              ]
            }}}
          ]
        }
      },
      $where: function() {
        var blackListedComponents = ['select', 'address'];
        var form = this;
        var walkComponents = function(components) {
          for(var a = 0; a < components.length; a++) {
            // Check the current component, to see if its unique.
            var component = components[a];
            if (component.unique === true && blackListedComponents.indexOf(component.type) === -1) {
              return true;
            }

            // Check any column components.
            if (component.hasOwnProperty('columns')) {
              if (walkComponents(component.columns) === true) {
                return true;
              }
            }
            // Check any row components.
            if (component.hasOwnProperty('rows')) {
              if (walkComponents(component.rows) === true) {
                return true;
              }
            }
            // Check any component components.
            if (component.hasOwnProperty('components')) {
              if (walkComponents(component.components) === true) {
                return true;
              }
            }
          }
        };

        return (walkComponents(form.components) === true);
      }
    })
    .snapshot(true)
    .toArray(function(err, forms) {
      if (err) {
        return next(err);
      }

      return next(null, forms);
    });
  };

  /**
   * Fix each of the given forms, before continuing
   *
   * @param forms
   * @param next
   * @returns {*}
   */
  var fixEachForm = function(forms, next) {
    if (!forms || forms.length === 0) {
      debug.fixEachForm('No forms');
      return next();
    }

    debug.fixEachForm('Fixing Stage');
    async.each(forms, fixFormUniques, function(err) {
      if (err) {
        return next(err);
      }

      debug.fixEachForm('done');
      return next();
    });
  };

  async.waterfall([
    getFormsWithUniqueComponents,
    fixEachForm,
    getFormsWithUniqueComponentsInLayoutComponents,
    fixEachForm,
    getFormsWithPotentialUniqueComponentsInLayoutComponents,
    fixEachForm
  ], function(err) {
    if (err) {
      return done(err);
    }

    done();
  });
};<commit_msg>Making update script more performant.<commit_after>'use strict';

var async = require('async');
var _ = require('lodash');
var debug = {
  getFormsWithUniqueComponents: require('debug')('formio:update:3.0.5-getFormsWithUniqueComponents'),
  getFormsWithUniqueComponentsInLayoutComponents: require('debug')('formio:update:3.0.5-getFormsWithUniqueComponentsInLayoutComponents'),
  getFormsWithPotentialUniqueComponentsInLayoutComponents: require('debug')('formio:update:3.0.5-getFormsWithPotentialUniqueComponentsInLayoutComponents'),
  getAffectedSubmissions: require('debug')('formio:update:3.0.5-getAffectedSubmissions'),
  buildUniqueComponentList: require('debug')('formio:update:3.0.5-buildUniqueComponentList'),
  fixSubmissionUniques: require('debug')('formio:update:3.0.5-fixSubmissionUniques')
};

/**
 * Update 3.0.5
 *
 * This update does the following.
 *
 *   1.) Finds all forms that have components with unique properties
 *   2.) Coerces all unique fields to be comparable
 *
 * @param db
 * @param config
 * @param tools
 * @param done
 */
module.exports = function(db, config, tools, done) {
  var formCollection = db.collection('forms');
  var submissionCollection = db.collection('submissions');
  var blackListedComponents = ['select', 'address'];

  // List of forms to sweep.
  var forms = [];

  // Unique map for all sweepable forms.
  var uniques = {};

  /**
   * Fix the submissions unique fields.
   *
   * @param submission
   * @param uniques
   * @param next
   * @returns {*}
   */
  var fixSubmissionUniques = function(submissions, next) {
    async.each(submissions, function(submission, cb) {
      var update = {};
      _.each(uniques[submission.form.toString()], function(path, key) {
        var item = _.get(submission, 'data.' + path);
        if (item) {
          // Coerce all unique string fields to be lowercase.
          if (typeof item === 'string') {
            debug.fixSubmissionUniques(submission._id.toString())
            update['data.' + path] = item.toString().toLowerCase();
          }
          // Coerce all unique string fields in an array to be lowercase.
          else if (item instanceof Array && (item.length > 0) && (typeof item[0] === 'string')) {
            _.map(item, function(element) {
              return element.toString().toLowerCase();
            });

            // Coerce all unique string fields to be lowercase.
            update['data.' + path] = item;
          }
        }
      });

      if (Object.keys(update).length === 0) {
        return cb();
      }

      submissionCollection.update(
        {_id: tools.util.idToBson(submission._id)},
        {$set: update},
        function(err) {
          if (err) {
            return cb(err);
          }

          return cb();
        }
      );
    }, function(err) {
      if (err) {
        return next(err);
      }

      return next();
    });
  };

  /**
   * Sweep all the forms to get all their uniques.
   *
   * @param next
   * @returns {*}
   */
  var buildUniqueComponentList = function(next) {
    formCollection.find({_id: {$in: forms}, deleted: {$eq: null}})
      .snapshot(true)
      .forEach(function(form) {
        tools.util.eachComponent(form.components, function(component, path) {
          // We only care about non-layout components, which are not unique, and have not been blacklisted.
          if (
            _.get(component, 'key')
            && _.get(component, 'unique') === true
            && blackListedComponents.indexOf(_.get(component, 'type')) === -1
          ) {
            debug.buildUniqueComponentList(form._id.toString() + ' -> ' + path);
            uniques[form._id.toString()] = uniques[form._id.toString()] || {};
            uniques[form._id.toString()][component.key] = path;
          }
        }, true);
      }, next());
  };

  /**
   * Get all the forms with a unique component in its root components list.
   *
   * @param next
   */
  var getFormsWithUniqueComponents = function(next) {
    formCollection.find({
      components: {$elemMatch: {unique: true, type: {$nin: blackListedComponents}}},
      deleted: {$eq: null}
    }, {_id: 1})
    .snapshot(true)
    .map(function(form) {
      return form._id;
    })
    .toArray(function(err, forms) {
      if (err) {
        return next(err);
      }

      debug.getFormsWithUniqueComponents(forms.length);
      return next(null, forms);
    });
  };

  /**
   * Get all the forms with a unique component within a layout component, that hasnt already been modified.
   *
   * @param next
   */
  var getFormsWithUniqueComponentsInLayoutComponents = function(next) {
    formCollection.find({
      _id: {$nin: forms},
      deleted: {$eq: null},
      components: {
        $elemMatch: {
          $or: [
            {$and: [{columns: {$exists: true}}, {columns: {$elemMatch: {unique: true, type: {$nin: blackListedComponents}}}}]},
            {$and: [{rows: {$exists: true}}, {rows: {$elemMatch: {unique: true, type: {$nin: blackListedComponents}}}}]},
            {$and: [{components: {$exists: true}}, {components: {$elemMatch: {unique: true, type: {$nin: blackListedComponents}}}}]}
          ]
        }
      }
    }, {_id: 1})
    .snapshot(true)
    .map(function(form) {
      return form._id;
    })
    .toArray(function(err, forms) {
      if (err) {
        return next(err);
      }

      debug.getFormsWithUniqueComponentsInLayoutComponents(forms.length);
      return next(null, forms);
    });
  };

  /**
   * Get all the forms with a layout component in a layout component, that hasnt already been modified.
   *
   * @param next
   */
  var getFormsWithPotentialUniqueComponentsInLayoutComponents = function(next) {
    formCollection.find({
      _id: {$nin: forms},
      deleted: {$eq: null},
      components: {
        $elemMatch: {
          $or: [
            {columns: {$elemMatch: {
              $or: [
                {columns: {$exists: true}},
                {rows: {$exists: true}},
                {components: {$exists: true}}
              ]
            }}},
            {rows: {$elemMatch: {
              $or: [
                {columns: {$exists: true}},
                {rows: {$exists: true}},
                {components: {$exists: true}}
              ]
            }}},
            {components: {$elemMatch: {
              $or: [
                {columns: {$exists: true}},
                {rows: {$exists: true}},
                {components: {$exists: true}}
              ]
            }}}
          ]
        }
      },
      $where: function() {
        var blackListedComponents = ['select', 'address'];
        var form = this;
        var walkComponents = function(components) {
          for(var a = 0; a < components.length; a++) {
            // Check the current component, to see if its unique.
            var component = components[a];
            if (component.unique === true && blackListedComponents.indexOf(component.type) === -1) {
              return true;
            }

            // Check any column components.
            if (component.hasOwnProperty('columns')) {
              if (walkComponents(component.columns) === true) {
                return true;
              }
            }
            // Check any row components.
            if (component.hasOwnProperty('rows')) {
              if (walkComponents(component.rows) === true) {
                return true;
              }
            }
            // Check any component components.
            if (component.hasOwnProperty('components')) {
              if (walkComponents(component.components) === true) {
                return true;
              }
            }
          }
        };

        return (walkComponents(form.components) === true);
      }
    }, {_id: 1})
    .snapshot(true)
    .map(function(form) {
      return form._id;
    })
    .toArray(function(err, forms) {
      if (err) {
        return next(err);
      }

      debug.getFormsWithPotentialUniqueComponentsInLayoutComponents(forms.length);
      return next(null, forms);
    });
  };

  /**
   * Get all submissions for modification.
   *
   * @param next
   */
  var getAffectedSubmissions = function(next) {
    submissionCollection.find({
      deleted: {$eq: null},
      form: {$in: forms}
    })
    .snapshot(true)
    .toArray(function(err, submissions) {
      if (err) {
        return next(err);
      }

      debug.getAffectedSubmissions(submissions.length);
      return next(null, submissions);
    });
  };

  /**
   * Build the list of forms.
   *
   * @param newForms
   * @param next
   */
  var mergeForms = function(newForms, next) {
    forms = forms.concat(newForms);
    next();
  };

  async.waterfall([
    getFormsWithUniqueComponents,
    mergeForms,
    getFormsWithUniqueComponentsInLayoutComponents,
    mergeForms,
    getFormsWithPotentialUniqueComponentsInLayoutComponents,
    mergeForms,
    buildUniqueComponentList,
    getAffectedSubmissions,
    fixSubmissionUniques
  ], function(err) {
    if (err) {
      return done(err);
    }

    done();
  });
};
{'fraction_non_alphanumeric': 0.14398432003216405, 'fraction_numerical': 0.0027138405869936676, 'mean_word_length': 2.1189655172413793, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 6, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4297785', 'n_tokens_mistral': 5725, 'n_tokens_neox': 5587, 'n_words': 1344}