<commit_before>var async = require('async');
var _ = require('lodash');

var diff = require('jsondiffpatch').create({
  objectHash: function(obj, index) {
    // try to find an id property, otherwise compare full JSON, which rules
    // out distinguishing content change from being an entirely new thing
    return obj._id || obj.id || JSON.stringify(obj);
  },
  textDiff: {
    // Don't try to diff text — replace it. Otherwise
    // patches are never applicable across locales
    minLength: 1000000000
  },
  arrays: {
    detectMove: true,
    // We don't actually copy the old value, however it is useful for the visual diff
    includeValueOnMove: true
  }
});

module.exports = {

  // ## Options
  // 
  // `includeTypes: [ 'my-blog-post', 'my-event' ]`
  // 
  // Apply workflow only to docs of the specified types. IF WORKFLOW IS ENABLED FOR ANY PAGE TYPE,
  // AS OPPOSED TO A PIECE, IT MUST BE ENABLED FOR *ALL* PAGE TYPES.
  // 
  // `excludeTypes: [ 'my-personal-profile' ]`
  // 
  // Apply workflow to everything EXCEPT the specified types. IF WORKFLOW IS ENABLED FOR ANY PAGE TYPE,
  // AS OPPOSED TO A PIECE, IT MUST BE ENABLED FOR *ALL* PAGE TYPES.
  // 
  // If both options are present, a type must appear in `includeTypes`
  // and NOT appear in `excludeTypes`.
  // 
  // `baseExcludeTypes: [ 'apostrophe-user', 'apostrophe-group' ]`
  // 
  // **Typically not changed.** A short list of types that should never be subject to workflow,
  // no matter what the other options say. For security reasons this list contains users and groups
  // by default. You will usually leave this alone.
  // 
  // `excludeProperties: [ 'hitCounter' ]`
  // 
  // A list of properties that should not be subject to workflow, but rather should be allowed to
  // vary for each locale and never be copied. These are typically properties
  // that don't make sense to edit as a "draft" and then submit as the new live version. For
  // instance, you wouldn't want to overwrite a page view counter field.
  // 
  // There is no `includeProperties` option. In Apostrophe 2.x workflow applies to properties by default,
  // and excluded properties are unique to the locale (that is, either draft or live version of the doc).
  //
  // `baseExcludeProperties`
  //
  // Like `baseExcludeTypes`, this overrides a short list of properties that must not be modified
  // by workflow. You don't want to change this.

  afterConstruct: function(self, callback) {
    self.extendCursor();
    self.extendIndexes();
    self.enableAddMissingLocalesTask();
    self.pushAssets();
    self.addToAdminBar();
    self.extendPieces();
    self.extendPermissionsField();
    self.extendPermissions();
    self.apos.pages.addAfterContextMenu(self.menu);
    return self.enableCollection(callback);
  },

  construct: function(self, options) {
    self.nestedLocales = options.locales || [
      {
        name: 'default'
      }
    ];
    
    self.locales = {};
    flattenLocales(self.nestedLocales);
    addDraftLocales();
    
    function flattenLocales(locales) {
      _.each(locales, function(locale) {
        self.locales[locale.name] = locale;
        if (locale.children) {
          flattenLocales(locale.children);
        }
      });
    }
    
    function addDraftLocales() {
      var newLocales = {};
      _.each(self.locales, function(locale, name) {
        newLocales[name] = locale;
        var draftLocale = _.cloneDeep(locale);
        draftLocale.name += '-draft';
        delete draftLocale.children;
        newLocales[draftLocale.name] = draftLocale;
      });
      self.locales = newLocales;
    }
    
    self.defaultLocale = options.defaultLocale || 'default';

    self.baseExcludeProperties = options.baseExcludeProperties || [
      '_id',
      'path',
      'rank',
      'level',
      'docPermissions',
      'createdAt',
      'updatedAt',
      'lowSearchText',
      'highSearchText',
      'highSearchWords',
      'searchSummary'
    ];
    
    // Attachment fields themselves are not directly localized (they are not docs)
    self.excludeActions = (self.options.excludeActions || []).concat(self.options.baseExcludeActions || [ 'admin', 'edit-attachment' ]);
    
    self.includeVerbs = (self.options.includeVerbs || []).concat(self.options.baseIncludeVerbs  || [ 'admin', 'edit' ]);

    // Localizing users and groups raises serious security questions. If they have a public representation,
    // make a new doc type and join to it
    self.baseExcludeTypes = [ 'apostrophe-user', 'apostrophe-group' ];

    // In 2.x, workflow applies to every property not explicitly excluded,
    // so configuration is simpler (localization will refine this though)
    self.excludeProperties = self.baseExcludeProperties.concat(options.excludeProperties || []);
    
    self.includeTypes = options.includeTypes || false;
    self.excludeTypes = self.baseExcludeTypes.concat(options.excludeTypes || []);

    // Extend all apostrophe cursors to limit results to the current locale by default
    self.extendCursor = function() {
      self.apos.define('apostrophe-cursor', require('./lib/cursor.js'));
    };

    require('./lib/permissions-schema-field.js')(self, options);
    
    // Extend the index parameters for the unique indexes on path and slug to allow for
    // two docs with the same slug in different locales

    self.extendIndexes = function() {
      self.apos.on('slugIndex', function(params) {
        params.workflowLocale = 1;
      });
      self.apos.on('pathIndex', function(params) {
        // Exactly like workflowLocale in every way except it exists only when
        // path exists. This allows the sparse index to work properly
        params.workflowLocaleForPathIndex = 1;
      });
    };
    
    self.extendPermissions = function() {
      self.apos.on('can', _.partial(self.onPermissions, 'can'));
      self.apos.on('criteria', _.partial(self.onPermissions, 'criteria'));
    };
    
    self.onPermissions = function(event, req, action, object, info) {
      if (_.contains(self.excludeActions, action)) {
        return;
      }
      if (!info.type) {
        return;
      }
      if (!self.apos.docs.getManager(info.type)) {
        return;
      }
      if (!self.includeType(info.type)) {
        return;
      }
      var verb = info.verb;
      // publish is not a separate verb in workflow since we already control whether you can edit
      // in draft vs. live locales
      if (verb === 'publish') {
        verb = 'edit';
      }
      if (!_.contains(self.includeVerbs, verb)) {
        return;
      }
      if (req.user && req.user._permissions.admin) {
        // Sitewide admins aren't restricted by locale because they can edit
        // groups, which would allow them to defeat that anyway
        return;
      }

      // OK, now we know this is something we're entitled to an opinion about

      // Rebuild the action string using the effective verb and type name
      action = info.verb + '-' + info.type;

      if (!(req.user && req.user._permissionsLocales && req.user._permissionsLocales[action])) {
        info.response = info._false;
        return;
      }

      var permissionsLocales = req.user._permissionsLocales[action];

      if (_.isEmpty(permissionsLocales)) {
        info.response = info._false;
        return;
      }

      if (event === 'criteria') {
        info.response = { $and: [ info.response, { workflowLocale: { $in: _.keys(permissionsLocales) } } ] };
      } else {
        var object = info.object || info.newObject;
        if (!permissionsLocales[object ? object.workflowLocale : req.locale ]) {
          info.response = info._false;
          return;
        }
      }

    };
    
    // When editing pieces, we should always get the draft version of
    // the content unless otherwise specified

    self.extendPieces = function() {
      self.apos.on('piecesFindForEditing', function(type, cursor) {
        if (!self.includeType(type)) {
          return;
        }
        var req = cursor.get('req');
        if (!req.locale.match(/\-draft$/)) {
          var locale = cursor.get('workflowLocale');
          if (locale === undefined) {
            cursor.workflowLocale(req.locale + '-draft');
          }
        }
      });
      self.apos.on('piecesEditControls', function(info) {
        upgradeControls(info);
      });
      self.apos.on('piecesCreateControls', function(info) {
        upgradeControls(info);
      });
      self.apos.on('pagesEditControls', function(info) {
        upgradeControls(info);
      });
      function upgradeControls(info) {
        if (!self.includeType(info.type)) {
          // Not subject to workflow
          return;
        }
        // TODO use info.req, check whether committing is a thing they can do
        // per Stuart's notes on permissions design.
        //
        // Also Submit operation.
        var save = _.find(info.controls, { action: 'save' });
        if (save) {
          save.label = 'Save Draft';
        }
        info.controls.push({
          type: 'major',
          label: 'Submit',
          action: 'submit'
        });
        // TODO: if and only if they are admin of this piece type in corresponding non-draft locale
        info.controls.push({
          type: 'major',
          label: 'Commit',
          action: 'commit'
        });
      }
    };

    // Every time a doc is saved, check whether its type is included in workflow. If it is,
    // and the doc does not yet have a `workflowLocale` property, establish one and generate
    // the `workflowGuid` property. Set the `_workflowNew` flag for the attention of
    // `docAfterSave`.

    self.docBeforeSave = function(req, doc, options) {
      if (doc._workflowPropagating) {
        // Recursion guard
        return;
      }
      if (!self.includeType(doc.type)) {
        return;
      }
      if (!doc.workflowLocale) {
        doc.workflowLocale = req.locale || self.defaultLocale;
        if (!doc.workflowLocale.match(/\-draft$/)) {
          // Always create the draft first, so we can then find it by id successfully
          // via code that is overridden to look for drafts. All the locales get created
          // but we want to return the draft's _id
          doc.workflowLocale += '-draft';
        }
        doc.workflowGuid = self.apos.utils.generateId();
        doc._workflowNew = true;
        self.ensureWorkflowLocaleForPathIndex(doc);
      }
    };

    // Provide a duplicate locale property, but only on pages, not pieces.
    // This enables us to use a sparse unique mongodb index. The property
    // should never be used for any other purpose.

    self.ensureWorkflowLocaleForPathIndex = function(doc) {
      if (doc.slug.match(/^\//)) {
        doc.workflowLocaleForPathIndex = doc.workflowLocale;
      }
    };
    
    // Every time a doc is saved, check whether its type is included in workflow. If it is,
    // check for locales in which that workflowGuid does not exist yet, and bring it into existence
    // there. If the doc has the `_workflowNew` property as set by `docBeforeSave`, we can assume
    // it is new in all other locales, otherwise query to find out.
    //
    // These newly created docs in other locales are initially unpublished.

    self.docAfterSave = function(req, doc, options, callback) {

      var missingLocales;
      
      if (doc._workflowPropagating) {
        // Recursion guard
        return callback(null);
      }
      
      if (!self.includeType(doc.type)) {
        return callback(null);
      }

      return async.series([
        findMissingLocales,
        insertInMissingLocales,
        permissionsAcrossLocales
      ], function(err) {
        if (err) {
          console.error(err);
        }
        return callback(err);
      });

      function findMissingLocales(callback) {
        if (doc._workflowNew) {
          missingLocales = _.filter(_.keys(self.locales), function(locale) {
            return locale !== doc.workflowLocale;
          });
          return callback(null);
        }
        return self.apos.docs.db.find({ workflowGuid: doc.workflowGuid }, { workflowLocale: 1 }).toArray(function(err, docs) {
          if (err) {
            return callback(err);
          }
          var locales = _.pluck(docs, 'workflowLocale');
          missingLocales = _.filter(_.keys(self.locales), function(locale) {
            return (locale !== doc.workflowLocale) && (!_.contains(locales, locale));
          });
          return callback(null);
        });
      }

      function insertInMissingLocales(callback) {
        if (!missingLocales.length) {
          return callback(null);
        }
        // A new doc needs to be brought into existence across all locales
        return async.eachSeries(_.keys(self.locales), function(locale, callback) {

          var _doc = self.apos.utils.clonePermanent(doc);
          if (locale === doc.workflowLocale) {
            return setImmediate(callback);
          }
          delete _doc._workflowNew;
          delete _doc._id;
          _doc.workflowLocale = locale;
          _doc._workflowPropagating = true;
          if (!locale.match(/\-draft$/)) {
            // Otherwise you can make something happen in public just by creating a new doc
            // that is published as a draft and watching it propagate
            _doc.published = false;
          }
          self.ensureWorkflowLocaleForPathIndex(_doc);
          return async.series([
            _.partial(self.resolveRelationships, req, _doc, _doc.workflowLocale),
            insert
          ], callback);
          
          function insert(callback) {
            // TODO: copy attachments so they are not directly shared resulting in cross-locale modification
            return self.apos.docs.insert(req, _doc, { permissions: false }, function(err) {
              return callback(err);
            });
          }

        }, callback);
      }
      
      function permissionsAcrossLocales(callback) {
        // If I can edit a specific page in ch-fr, I can also edit that same page in gb-en,
        // PROVIDED THAT I can edit pages in gb-en at all (we have locale-specific
        // permission checks). This eliminates complexities in the permissions interface.
        if (!doc.docPermissions) {
          return callback(null);
        }
        return self.apos.docs.db.update({
          workflowGuid: doc.workflowGuid
        }, {
          $set: {
            docPermissions: doc.docPermissions
          }
        }, {
          multi: true
        }, callback);
      }
    };
    
    // Resolve relationships between this doc and other docs, which need to be
    // mapped to the appropriate doc in the new locale, via the workflowGuid
    // property of each doc.
    //
    // Existing join ids in `doc` are remapped to the corresponding ids in `toLocale`.
    //
    // This method DOES NOT save the doc and should not modify anything on its own.
    // Note that the diff implementation utilizes this method so that it's comparing
    // ids of the same locale.

    self.resolveRelationships = function(req, doc, toLocale, callback) {

      // Expansion is anticipated

      return async.series([
        mapJoins
      ], function(err) {
        return callback(err);
      });

      function mapJoins(callback) {
        // First create an array of objects with doc and field properties, so we can asynchronously
        // iterate over them

        var joins = [];
        var workflowGuidToOldId = {};
        var oldIdToNewId = {};

        findJoinsInDocSchema();
        findJoinsInAreas();

        var workflowGuids;
        var secondLocaleIds;
        var originalIds;

        return async.eachSeries(joins, function(join, callback) {
          return async.series([
            findWorkflowGuids,
            findSecondLocaleIds
          ], function(err) {
            if (err) {
              return callback(err);
            }
            remapDocs();
            return callback(null);
          });
          
          function findWorkflowGuids(callback) {
            if (join.type === 'joinByOne') {
              return self.apos.docs.db.find({ _id: { $in: [ join.doc[join.field.idField] ] } }, { workflowGuid: 1 }).toArray(function(err, docs) {
                if (err) {
                  return callback(err);
                }
                workflowGuids = _.pluck(docs, 'workflowGuid');
                _.each(docs, function(doc) {
                  workflowGuidToOldId[doc.workflowGuid] = doc._id;
                });
                return callback(null);
              });
            } else {
              return self.apos.docs.db.find({ _id: { $in: join.doc[join.field.idsField] || [] } }, { workflowGuid: 1 }).toArray(function(err, docs) {
                originalIds = join.doc[join.field.idsField];
                if (err) {
                  return callback(err);
                }
                _.each(docs, function(doc) {
                  workflowGuidToOldId[doc.workflowGuid] = doc._id;
                });
                workflowGuids = _.pluck(docs, 'workflowGuid');
                return callback(null);
              });
            }
          }

          function findSecondLocaleIds(callback) {
            return self.apos.docs.db.find({ workflowGuid: { $in: workflowGuids }, workflowLocale: toLocale }, { _id: 1, workflowGuid: 1 }).toArray(function(err, docs) {
              if (err) {
                return callback(err);
              }
              secondLocaleIds = _.pluck(docs, '_id');
              _.each(docs, function(doc) {
                if (_.has(workflowGuidToOldId, doc.workflowGuid)) {
                  oldIdToNewId[workflowGuidToOldId[doc.workflowGuid]] = doc._id;
                }
              });
              return callback(null);
            });
          }

          function remapDocs() {
            if (join.type === 'joinByOne') {
              join.doc[join.field.idField] = secondLocaleIds[0];
            } else {
              if (join.field.relationship) {
                var relationships = join.doc[join.field.relationshipsField];
                var newRelationships = {};
                _.each(relationships, function(val, _id) {
                  if (_.has(oldIdToNewId, _id)) {
                    newRelationships[oldIdToNewId[_id]] = val;
                  }
                });
                join.doc[join.field.relationshipsField] = newRelationships;
              }
              // Rebuild the original order properly
              secondLocaleIds = _.map(originalIds, function(id) {
                return oldIdToNewId[id];
              });
              join.doc[join.field.idsField] = secondLocaleIds;
            }
          }

        }, callback);
                  
        function findJoinsInDocSchema() {
          var schema = self.apos.docs.getManager(doc.type).schema;
          joins = joins.concat(findJoinsInSchema(doc, schema));
        }
        
        function findJoinsInAreas() {
          var widgets = [];
          self.apos.areas.walk(doc, function(area, dotPath) {
            widgets = widgets.concat(area.items);
          });
          widgets = _.filter(widgets, function(widget) {
            var schema = self.apos.areas.getWidgetManager(widget.type).schema;
            joins = joins.concat(findJoinsInSchema(widget, schema));
          });
        }
        
        function findJoinsInSchema(doc, schema) {
          return _.map(
            _.filter(
              schema, function(field) {
                if ((field.type === 'joinByOne') || (field.type === 'joinByArray')) {
                  if (self.includeType(field.withType)) {
                    return true;
                  }
                }
              }
            ), function(field) {
              return { doc: doc, field: field };
            }
          );
        }  
      } 
   
    };

    // Commit a doc from one locale to another. `from` and `to` should
    // be the doc as found in each of the locales. The callback receives
    // `(null, commitId)` where `commitId` is a unique identifier for
    // this specific commit.

    self.commit = function(req, from, to, callback) {
      delete from.workflowSubmitted;
      var commitId;
      // For storage in the commits collection
      var originalTo = self.apos.utils.clonePermanent(to);
      return async.series([
        // Resolve the relationships for originalTo as well so we can straightforwardly
        // call diff() later
        _.partial(self.resolveRelationships, req, originalTo, to.workflowLocale),
        _.partial(self.copyIncludedProperties, req, from, to),
        _.partial(self.resolveRelationships, req, to, to.workflowLocale),
        insertCommit,
        _.partial(self.apos.docs.update, req, to)
      ], function(err) {
        return callback(err, commitId);
      });
      function insertCommit(callback) {
        return self.insertCommit(req, from, originalTo, function(err, _commitId) {
          if (err) {
            return callback(err);
          }
          commitId = _commitId;
          return callback(null);
        });
      }
    };
    
    // Fetch the draft version of a doc, whose id is `id`, and also the live version of the same
    // doc. On success, deliver `(null, draft, live)` to the callback.
    //
    // This method will operate properly regardless of whether `req.locale` is the live locale
    // or the one with the `-draft` suffix.
    
    self.getDraftAndLive = function(req, id, callback) {
      var draft;
      var live;
      return async.series([
        getDraft,
        getLive
      ], function(err) {
        if (err) {
          return callback(err);
        }
        return callback(null, draft, live);
      });
      function getDraft(callback) {
        var locale = req.locale;
        if (!req.locale.match(/\-draft$/)) {
          locale += '-draft';
        }
        return self.apos.docs.find(req, { _id: id }).permission('edit').published(null).workflowLocale(locale).toObject(function(err, _draft) {
          if (err) {
            return callback(err);
          }
          draft = _draft;
          if (!draft) {
            return callback('draft not found');
          }
          return callback(null);
        });
      }
      // We don't actually need the live version (the previewing stuff happens in an iframe),
      // but we should verify we have edit permissions there
      function getLive(callback) {
        return self.apos.docs.find(req, { workflowGuid: draft.workflowGuid }).workflowLocale(req.locale.replace(/\-draft$/, '')).permission('edit').published(null).toObject(function(err, _live) {
          if (err) {
            return callback(err);
          }
          live = _live;
          if (!live) {
            return callback('live not found');
          }
          return callback(null);
        });
      }      
    };
    
    self.route('post', 'commit', function(req, res) {
      if (!req.user) {
        // Confusion to the enemy
        return res.status(404).send('not found');
      }
      var id = self.apos.launder.id(req.body.id);
      var draft, live, commitId;
      return async.series({
        getDraftAndLive,
        commit
      }, function(err) {
        if (err) {
          console.error(err);
          return res.send({ status: 'error' });
        }
        return res.send({ status: 'ok', commitId: commitId });
      });
      function getDraftAndLive(callback) {
        return self.getDraftAndLive(req, id, function(err, _draft, _live) {
          if (err) {
            return callback(err);
          }
          draft = _draft;
          live = _live;
          return callback(null, draft, live);
        });
      }
      function commit(callback) {
        return self.commit(req, draft, live, function(err, _commitId) {
          commitId = _commitId;
          return callback(err);
        });
      }
    });

    self.route('post', 'export', function(req, res) {
      if (!req.user) {
        // Confusion to the enemy
        return res.status(404).send('not found');
      }
      var id = self.apos.launder.id(req.body.id);
      var locales = [];
      var errors = [];
      var drafts;
      if (Array.isArray(req.body.locales)) {
        locales = _.filter(req.body.locales, function(locale) {
          return ((typeof(locale) === 'string') && (_.has(self.locales, locale)));
        });
      }
      locales = _.map(locales, function(locale) {
        return locale + '-draft';
      });
      var commit;
      return async.series({
        getCommit,
        applyPatches
      }, function(err) {
        if (err) {
          console.error(err);
          return res.send({ status: 'error' });
        }
        // Here the `errors` object has locales as keys and strings as values; these can be
        // displayed or, since they are technical, just flagged as locales to which the patch
        // could not be successfully applied
        return res.send({ status: 'ok', errors: errors });
      });
      function getCommit(callback) {
        return self.findDocAndCommit(req, id, function(err, doc, _commit) {
          if (err) {
            return callback(err);
          }
          commit = _commit;
          if (!commit) {
            return callback('notfound');
          }
          return callback(null);
        });
      }
      function applyPatches(callback) {
        return async.eachSeries(locales, function(locale, callback) {
          var draft;
          return async.series([ getDraft, resolveToSource, applyPatch, resolveToDestination, update ], callback);
          function getDraft(callback) {
            return self.apos.docs.find(req, { workflowGuid: commit.workflowGuid }).published(null).workflowLocale(locale).permission('edit').toObject(function(err, _draft) {
              if (err) {
                return callback(err);
              }
              draft = _draft;
              return callback(null);
            });
          }
          // Resolve relationship ids to point to the locale the patch is coming from,
          // so that the diff applies properly
          function resolveToSource(callback) {
            return self.resolveRelationships(req, draft, commit.to.workflowLocale, callback);
          }
          function applyPatch(callback) {
            if (!draft) {
              errors.push(locale.replace(/\-draft$/, '') + ': not found, run task');
              return callback(null);
            }
            self.deleteExcludedProperties(commit.from);
            self.deleteExcludedProperties(commit.to);
            var patch = diff.diff(commit.to, commit.from);
            // If a patch edits a widget in an area that doesn't exist at all yet
            // in the receiving locale, create the area
            _.each(patch, function(value, key) {
              if (!draft[key]) {
                if (commit.from[key] && (commit.from[key].type === 'area')) {
                  draft[key] = {
                    type: 'area',
                    items: []
                  };
                }
              }
            });
            try {
              console.log('patch is: ', JSON.stringify(patch, null, '  '));
              console.log('draft is: ', JSON.stringify(draft, null, '  '));
              diff.patch(draft, patch);
            } catch (e) {
              errors.push(locale.replace(/\-draft$/, ''));
              console.error(e);
            }
            return callback(null);
          }
          // Resolve relationship ids back to the locale the draft is coming from
          function resolveToDestination(callback) {
            return self.resolveRelationships(req, draft, draft.workflowLocale, callback);
          }
          function update(callback) {
            return self.apos.docs.update(req, draft, callback); 
          }
        }, callback);
      }
    });
    
    self.deleteExcludedProperties = function(doc) {
      _.each(doc, function(val, key) {
        if (!self.includeProperty(key)) {
          delete doc[key];
        }<commit_msg>improved patching implemented, must now be tested<commit_after>var async = require('async');
var _ = require('lodash');
var deep = require('deep-get-set');

var diff = require('jsondiffpatch').create({
  objectHash: function(obj, index) {
    // try to find an id property, otherwise compare full JSON, which rules
    // out distinguishing content change from being an entirely new thing
    return obj._id || obj.id || JSON.stringify(obj);
  },
  textDiff: {
    // Don't try to diff text — replace it. Otherwise
    // patches are never applicable across locales
    minLength: 1000000000
  },
  arrays: {
    detectMove: true,
    // We don't actually copy the old value, however it is useful for the visual diff
    includeValueOnMove: true
  }
});

module.exports = {

  // ## Options
  // 
  // `includeTypes: [ 'my-blog-post', 'my-event' ]`
  // 
  // Apply workflow only to docs of the specified types. IF WORKFLOW IS ENABLED FOR ANY PAGE TYPE,
  // AS OPPOSED TO A PIECE, IT MUST BE ENABLED FOR *ALL* PAGE TYPES.
  // 
  // `excludeTypes: [ 'my-personal-profile' ]`
  // 
  // Apply workflow to everything EXCEPT the specified types. IF WORKFLOW IS ENABLED FOR ANY PAGE TYPE,
  // AS OPPOSED TO A PIECE, IT MUST BE ENABLED FOR *ALL* PAGE TYPES.
  // 
  // If both options are present, a type must appear in `includeTypes`
  // and NOT appear in `excludeTypes`.
  // 
  // `baseExcludeTypes: [ 'apostrophe-user', 'apostrophe-group' ]`
  // 
  // **Typically not changed.** A short list of types that should never be subject to workflow,
  // no matter what the other options say. For security reasons this list contains users and groups
  // by default. You will usually leave this alone.
  // 
  // `excludeProperties: [ 'hitCounter' ]`
  // 
  // A list of properties that should not be subject to workflow, but rather should be allowed to
  // vary for each locale and never be copied. These are typically properties
  // that don't make sense to edit as a "draft" and then submit as the new live version. For
  // instance, you wouldn't want to overwrite a page view counter field.
  // 
  // There is no `includeProperties` option. In Apostrophe 2.x workflow applies to properties by default,
  // and excluded properties are unique to the locale (that is, either draft or live version of the doc).
  //
  // `baseExcludeProperties`
  //
  // Like `baseExcludeTypes`, this overrides a short list of properties that must not be modified
  // by workflow. You don't want to change this.

  afterConstruct: function(self, callback) {
    self.extendCursor();
    self.extendIndexes();
    self.enableAddMissingLocalesTask();
    self.pushAssets();
    self.addToAdminBar();
    self.extendPieces();
    self.extendPermissionsField();
    self.extendPermissions();
    self.apos.pages.addAfterContextMenu(self.menu);
    return self.enableCollection(callback);
  },

  construct: function(self, options) {
    self.nestedLocales = options.locales || [
      {
        name: 'default'
      }
    ];
    
    self.locales = {};
    flattenLocales(self.nestedLocales);
    addDraftLocales();
    
    function flattenLocales(locales) {
      _.each(locales, function(locale) {
        self.locales[locale.name] = locale;
        if (locale.children) {
          flattenLocales(locale.children);
        }
      });
    }
    
    function addDraftLocales() {
      var newLocales = {};
      _.each(self.locales, function(locale, name) {
        newLocales[name] = locale;
        var draftLocale = _.cloneDeep(locale);
        draftLocale.name += '-draft';
        delete draftLocale.children;
        newLocales[draftLocale.name] = draftLocale;
      });
      self.locales = newLocales;
    }
    
    self.defaultLocale = options.defaultLocale || 'default';

    self.baseExcludeProperties = options.baseExcludeProperties || [
      '_id',
      'path',
      'rank',
      'level',
      'docPermissions',
      'createdAt',
      'updatedAt',
      'lowSearchText',
      'highSearchText',
      'highSearchWords',
      'searchSummary'
    ];
    
    // Attachment fields themselves are not directly localized (they are not docs)
    self.excludeActions = (self.options.excludeActions || []).concat(self.options.baseExcludeActions || [ 'admin', 'edit-attachment' ]);
    
    self.includeVerbs = (self.options.includeVerbs || []).concat(self.options.baseIncludeVerbs  || [ 'admin', 'edit' ]);

    // Localizing users and groups raises serious security questions. If they have a public representation,
    // make a new doc type and join to it
    self.baseExcludeTypes = [ 'apostrophe-user', 'apostrophe-group' ];

    // In 2.x, workflow applies to every property not explicitly excluded,
    // so configuration is simpler (localization will refine this though)
    self.excludeProperties = self.baseExcludeProperties.concat(options.excludeProperties || []);
    
    self.includeTypes = options.includeTypes || false;
    self.excludeTypes = self.baseExcludeTypes.concat(options.excludeTypes || []);

    // Extend all apostrophe cursors to limit results to the current locale by default
    self.extendCursor = function() {
      self.apos.define('apostrophe-cursor', require('./lib/cursor.js'));
    };

    require('./lib/permissions-schema-field.js')(self, options);
    
    // Extend the index parameters for the unique indexes on path and slug to allow for
    // two docs with the same slug in different locales

    self.extendIndexes = function() {
      self.apos.on('slugIndex', function(params) {
        params.workflowLocale = 1;
      });
      self.apos.on('pathIndex', function(params) {
        // Exactly like workflowLocale in every way except it exists only when
        // path exists. This allows the sparse index to work properly
        params.workflowLocaleForPathIndex = 1;
      });
    };
    
    self.extendPermissions = function() {
      self.apos.on('can', _.partial(self.onPermissions, 'can'));
      self.apos.on('criteria', _.partial(self.onPermissions, 'criteria'));
    };
    
    self.onPermissions = function(event, req, action, object, info) {
      if (_.contains(self.excludeActions, action)) {
        return;
      }
      if (!info.type) {
        return;
      }
      if (!self.apos.docs.getManager(info.type)) {
        return;
      }
      if (!self.includeType(info.type)) {
        return;
      }
      var verb = info.verb;
      // publish is not a separate verb in workflow since we already control whether you can edit
      // in draft vs. live locales
      if (verb === 'publish') {
        verb = 'edit';
      }
      if (!_.contains(self.includeVerbs, verb)) {
        return;
      }
      if (req.user && req.user._permissions.admin) {
        // Sitewide admins aren't restricted by locale because they can edit
        // groups, which would allow them to defeat that anyway
        return;
      }

      // OK, now we know this is something we're entitled to an opinion about

      // Rebuild the action string using the effective verb and type name
      action = info.verb + '-' + info.type;

      if (!(req.user && req.user._permissionsLocales && req.user._permissionsLocales[action])) {
        info.response = info._false;
        return;
      }

      var permissionsLocales = req.user._permissionsLocales[action];

      if (_.isEmpty(permissionsLocales)) {
        info.response = info._false;
        return;
      }

      if (event === 'criteria') {
        info.response = { $and: [ info.response, { workflowLocale: { $in: _.keys(permissionsLocales) } } ] };
      } else {
        var object = info.object || info.newObject;
        if (!permissionsLocales[object ? object.workflowLocale : req.locale ]) {
          info.response = info._false;
          return;
        }
      }

    };
    
    // When editing pieces, we should always get the draft version of
    // the content unless otherwise specified

    self.extendPieces = function() {
      self.apos.on('piecesFindForEditing', function(type, cursor) {
        if (!self.includeType(type)) {
          return;
        }
        var req = cursor.get('req');
        if (!req.locale.match(/\-draft$/)) {
          var locale = cursor.get('workflowLocale');
          if (locale === undefined) {
            cursor.workflowLocale(req.locale + '-draft');
          }
        }
      });
      self.apos.on('piecesEditControls', function(info) {
        upgradeControls(info);
      });
      self.apos.on('piecesCreateControls', function(info) {
        upgradeControls(info);
      });
      self.apos.on('pagesEditControls', function(info) {
        upgradeControls(info);
      });
      function upgradeControls(info) {
        if (!self.includeType(info.type)) {
          // Not subject to workflow
          return;
        }
        // TODO use info.req, check whether committing is a thing they can do
        // per Stuart's notes on permissions design.
        //
        // Also Submit operation.
        var save = _.find(info.controls, { action: 'save' });
        if (save) {
          save.label = 'Save Draft';
        }
        info.controls.push({
          type: 'major',
          label: 'Submit',
          action: 'submit'
        });
        // TODO: if and only if they are admin of this piece type in corresponding non-draft locale
        info.controls.push({
          type: 'major',
          label: 'Commit',
          action: 'commit'
        });
      }
    };

    // Every time a doc is saved, check whether its type is included in workflow. If it is,
    // and the doc does not yet have a `workflowLocale` property, establish one and generate
    // the `workflowGuid` property. Set the `_workflowNew` flag for the attention of
    // `docAfterSave`.

    self.docBeforeSave = function(req, doc, options) {
      if (doc._workflowPropagating) {
        // Recursion guard
        return;
      }
      if (!self.includeType(doc.type)) {
        return;
      }
      if (!doc.workflowLocale) {
        doc.workflowLocale = req.locale || self.defaultLocale;
        if (!doc.workflowLocale.match(/\-draft$/)) {
          // Always create the draft first, so we can then find it by id successfully
          // via code that is overridden to look for drafts. All the locales get created
          // but we want to return the draft's _id
          doc.workflowLocale += '-draft';
        }
        doc.workflowGuid = self.apos.utils.generateId();
        doc._workflowNew = true;
        self.ensureWorkflowLocaleForPathIndex(doc);
      }
    };

    // Provide a duplicate locale property, but only on pages, not pieces.
    // This enables us to use a sparse unique mongodb index. The property
    // should never be used for any other purpose.

    self.ensureWorkflowLocaleForPathIndex = function(doc) {
      if (doc.slug.match(/^\//)) {
        doc.workflowLocaleForPathIndex = doc.workflowLocale;
      }
    };
    
    // Every time a doc is saved, check whether its type is included in workflow. If it is,
    // check for locales in which that workflowGuid does not exist yet, and bring it into existence
    // there. If the doc has the `_workflowNew` property as set by `docBeforeSave`, we can assume
    // it is new in all other locales, otherwise query to find out.
    //
    // These newly created docs in other locales are initially unpublished.

    self.docAfterSave = function(req, doc, options, callback) {

      var missingLocales;
      
      if (doc._workflowPropagating) {
        // Recursion guard
        return callback(null);
      }
      
      if (!self.includeType(doc.type)) {
        return callback(null);
      }

      return async.series([
        findMissingLocales,
        insertInMissingLocales,
        permissionsAcrossLocales
      ], function(err) {
        if (err) {
          console.error(err);
        }
        return callback(err);
      });

      function findMissingLocales(callback) {
        if (doc._workflowNew) {
          missingLocales = _.filter(_.keys(self.locales), function(locale) {
            return locale !== doc.workflowLocale;
          });
          return callback(null);
        }
        return self.apos.docs.db.find({ workflowGuid: doc.workflowGuid }, { workflowLocale: 1 }).toArray(function(err, docs) {
          if (err) {
            return callback(err);
          }
          var locales = _.pluck(docs, 'workflowLocale');
          missingLocales = _.filter(_.keys(self.locales), function(locale) {
            return (locale !== doc.workflowLocale) && (!_.contains(locales, locale));
          });
          return callback(null);
        });
      }

      function insertInMissingLocales(callback) {
        if (!missingLocales.length) {
          return callback(null);
        }
        // A new doc needs to be brought into existence across all locales
        return async.eachSeries(_.keys(self.locales), function(locale, callback) {

          var _doc = self.apos.utils.clonePermanent(doc);
          if (locale === doc.workflowLocale) {
            return setImmediate(callback);
          }
          delete _doc._workflowNew;
          delete _doc._id;
          _doc.workflowLocale = locale;
          _doc._workflowPropagating = true;
          if (!locale.match(/\-draft$/)) {
            // Otherwise you can make something happen in public just by creating a new doc
            // that is published as a draft and watching it propagate
            _doc.published = false;
          }
          self.ensureWorkflowLocaleForPathIndex(_doc);
          return async.series([
            _.partial(self.resolveRelationships, req, _doc, _doc.workflowLocale),
            insert
          ], callback);
          
          function insert(callback) {
            // TODO: copy attachments so they are not directly shared resulting in cross-locale modification
            return self.apos.docs.insert(req, _doc, { permissions: false }, function(err) {
              return callback(err);
            });
          }

        }, callback);
      }
      
      function permissionsAcrossLocales(callback) {
        // If I can edit a specific page in ch-fr, I can also edit that same page in gb-en,
        // PROVIDED THAT I can edit pages in gb-en at all (we have locale-specific
        // permission checks). This eliminates complexities in the permissions interface.
        if (!doc.docPermissions) {
          return callback(null);
        }
        return self.apos.docs.db.update({
          workflowGuid: doc.workflowGuid
        }, {
          $set: {
            docPermissions: doc.docPermissions
          }
        }, {
          multi: true
        }, callback);
      }
    };
    
    // Resolve relationships between this doc and other docs, which need to be
    // mapped to the appropriate doc in the new locale, via the workflowGuid
    // property of each doc.
    //
    // Existing join ids in `doc` are remapped to the corresponding ids in `toLocale`.
    //
    // This method DOES NOT save the doc and should not modify anything on its own.
    // Note that the diff implementation utilizes this method so that it's comparing
    // ids of the same locale.

    self.resolveRelationships = function(req, doc, toLocale, callback) {

      // Expansion is anticipated

      return async.series([
        mapJoins
      ], function(err) {
        return callback(err);
      });

      function mapJoins(callback) {
        // First create an array of objects with doc and field properties, so we can asynchronously
        // iterate over them

        var joins = [];
        var workflowGuidToOldId = {};
        var oldIdToNewId = {};

        findJoinsInDocSchema();
        findJoinsInAreas();

        var workflowGuids;
        var secondLocaleIds;
        var originalIds;

        return async.eachSeries(joins, function(join, callback) {
          return async.series([
            findWorkflowGuids,
            findSecondLocaleIds
          ], function(err) {
            if (err) {
              return callback(err);
            }
            remapDocs();
            return callback(null);
          });
          
          function findWorkflowGuids(callback) {
            if (join.type === 'joinByOne') {
              return self.apos.docs.db.find({ _id: { $in: [ join.doc[join.field.idField] ] } }, { workflowGuid: 1 }).toArray(function(err, docs) {
                if (err) {
                  return callback(err);
                }
                workflowGuids = _.pluck(docs, 'workflowGuid');
                _.each(docs, function(doc) {
                  workflowGuidToOldId[doc.workflowGuid] = doc._id;
                });
                return callback(null);
              });
            } else {
              return self.apos.docs.db.find({ _id: { $in: join.doc[join.field.idsField] || [] } }, { workflowGuid: 1 }).toArray(function(err, docs) {
                originalIds = join.doc[join.field.idsField];
                if (err) {
                  return callback(err);
                }
                _.each(docs, function(doc) {
                  workflowGuidToOldId[doc.workflowGuid] = doc._id;
                });
                workflowGuids = _.pluck(docs, 'workflowGuid');
                return callback(null);
              });
            }
          }

          function findSecondLocaleIds(callback) {
            return self.apos.docs.db.find({ workflowGuid: { $in: workflowGuids }, workflowLocale: toLocale }, { _id: 1, workflowGuid: 1 }).toArray(function(err, docs) {
              if (err) {
                return callback(err);
              }
              secondLocaleIds = _.pluck(docs, '_id');
              _.each(docs, function(doc) {
                if (_.has(workflowGuidToOldId, doc.workflowGuid)) {
                  oldIdToNewId[workflowGuidToOldId[doc.workflowGuid]] = doc._id;
                }
              });
              return callback(null);
            });
          }

          function remapDocs() {
            if (join.type === 'joinByOne') {
              join.doc[join.field.idField] = secondLocaleIds[0];
            } else {
              if (join.field.relationship) {
                var relationships = join.doc[join.field.relationshipsField];
                var newRelationships = {};
                _.each(relationships, function(val, _id) {
                  if (_.has(oldIdToNewId, _id)) {
                    newRelationships[oldIdToNewId[_id]] = val;
                  }
                });
                join.doc[join.field.relationshipsField] = newRelationships;
              }
              // Rebuild the original order properly
              secondLocaleIds = _.map(originalIds, function(id) {
                return oldIdToNewId[id];
              });
              join.doc[join.field.idsField] = secondLocaleIds;
            }
          }

        }, callback);
                  
        function findJoinsInDocSchema() {
          var schema = self.apos.docs.getManager(doc.type).schema;
          joins = joins.concat(findJoinsInSchema(doc, schema));
        }
        
        function findJoinsInAreas() {
          var widgets = [];
          self.apos.areas.walk(doc, function(area, dotPath) {
            widgets = widgets.concat(area.items);
          });
          widgets = _.filter(widgets, function(widget) {
            var schema = self.apos.areas.getWidgetManager(widget.type).schema;
            joins = joins.concat(findJoinsInSchema(widget, schema));
          });
        }
        
        function findJoinsInSchema(doc, schema) {
          return _.map(
            _.filter(
              schema, function(field) {
                if ((field.type === 'joinByOne') || (field.type === 'joinByArray')) {
                  if (self.includeType(field.withType)) {
                    return true;
                  }
                }
              }
            ), function(field) {
              return { doc: doc, field: field };
            }
          );
        }  
      } 
   
    };

    // Commit a doc from one locale to another. `from` and `to` should
    // be the doc as found in each of the locales. The callback receives
    // `(null, commitId)` where `commitId` is a unique identifier for
    // this specific commit.

    self.commit = function(req, from, to, callback) {
      delete from.workflowSubmitted;
      var commitId;
      // For storage in the commits collection
      var originalTo = self.apos.utils.clonePermanent(to);
      return async.series([
        // Resolve the relationships for originalTo as well so we can straightforwardly
        // call diff() later
        _.partial(self.resolveRelationships, req, originalTo, to.workflowLocale),
        _.partial(self.copyIncludedProperties, req, from, to),
        _.partial(self.resolveRelationships, req, to, to.workflowLocale),
        insertCommit,
        _.partial(self.apos.docs.update, req, to)
      ], function(err) {
        return callback(err, commitId);
      });
      function insertCommit(callback) {
        return self.insertCommit(req, from, originalTo, function(err, _commitId) {
          if (err) {
            return callback(err);
          }
          commitId = _commitId;
          return callback(null);
        });
      }
    };
    
    // Fetch the draft version of a doc, whose id is `id`, and also the live version of the same
    // doc. On success, deliver `(null, draft, live)` to the callback.
    //
    // This method will operate properly regardless of whether `req.locale` is the live locale
    // or the one with the `-draft` suffix.
    
    self.getDraftAndLive = function(req, id, callback) {
      var draft;
      var live;
      return async.series([
        getDraft,
        getLive
      ], function(err) {
        if (err) {
          return callback(err);
        }
        return callback(null, draft, live);
      });
      function getDraft(callback) {
        var locale = req.locale;
        if (!req.locale.match(/\-draft$/)) {
          locale += '-draft';
        }
        return self.apos.docs.find(req, { _id: id }).permission('edit').published(null).workflowLocale(locale).toObject(function(err, _draft) {
          if (err) {
            return callback(err);
          }
          draft = _draft;
          if (!draft) {
            return callback('draft not found');
          }
          return callback(null);
        });
      }
      // We don't actually need the live version (the previewing stuff happens in an iframe),
      // but we should verify we have edit permissions there
      function getLive(callback) {
        return self.apos.docs.find(req, { workflowGuid: draft.workflowGuid }).workflowLocale(req.locale.replace(/\-draft$/, '')).permission('edit').published(null).toObject(function(err, _live) {
          if (err) {
            return callback(err);
          }
          live = _live;
          if (!live) {
            return callback('live not found');
          }
          return callback(null);
        });
      }      
    };
    
    self.route('post', 'commit', function(req, res) {
      if (!req.user) {
        // Confusion to the enemy
        return res.status(404).send('not found');
      }
      var id = self.apos.launder.id(req.body.id);
      var draft, live, commitId;
      return async.series({
        getDraftAndLive,
        commit
      }, function(err) {
        if (err) {
          console.error(err);
          return res.send({ status: 'error' });
        }
        return res.send({ status: 'ok', commitId: commitId });
      });
      function getDraftAndLive(callback) {
        return self.getDraftAndLive(req, id, function(err, _draft, _live) {
          if (err) {
            return callback(err);
          }
          draft = _draft;
          live = _live;
          return callback(null, draft, live);
        });
      }
      function commit(callback) {
        return self.commit(req, draft, live, function(err, _commitId) {
          commitId = _commitId;
          return callback(err);
        });
      }
    });

    self.route('post', 'export', function(req, res) {
      if (!req.user) {
        // Confusion to the enemy
        return res.status(404).send('not found');
      }
      var id = self.apos.launder.id(req.body.id);
      var locales = [];
      var errors = [];
      var drafts;
      if (Array.isArray(req.body.locales)) {
        locales = _.filter(req.body.locales, function(locale) {
          return ((typeof(locale) === 'string') && (_.has(self.locales, locale)));
        });
      }
      locales = _.map(locales, function(locale) {
        return locale + '-draft';
      });
      var commit;
      return async.series({
        getCommit,
        applyPatches
      }, function(err) {
        if (err) {
          console.error(err);
          return res.send({ status: 'error' });
        }
        // Here the `errors` object has locales as keys and strings as values; these can be
        // displayed or, since they are technical, just flagged as locales to which the patch
        // could not be successfully applied
        return res.send({ status: 'ok', errors: errors });
      });
      function getCommit(callback) {
        return self.findDocAndCommit(req, id, function(err, doc, _commit) {
          if (err) {
            return callback(err);
          }
          commit = _commit;
          if (!commit) {
            return callback('notfound');
          }
          return callback(null);
        });
      }
      function applyPatches(callback) {
        return async.eachSeries(locales, function(locale, callback) {
          var draft;
          return async.series([ getDraft, resolveToSource, applyPatch, resolveToDestination, update ], callback);
          function getDraft(callback) {
            return self.apos.docs.find(req, { workflowGuid: commit.workflowGuid }).published(null).workflowLocale(locale).permission('edit').toObject(function(err, _draft) {
              if (err) {
                return callback(err);
              }
              draft = _draft;
              return callback(null);
            });
          }
          // Resolve relationship ids to point to the locale the patch is coming from,
          // so that the diff applies properly
          function resolveToSource(callback) {
            return self.resolveRelationships(req, draft, commit.to.workflowLocale, callback);
          }
          function applyPatch(callback) {
            if (!draft) {
              errors.push(locale.replace(/\-draft$/, '') + ': not found, run task');
              return callback(null);
            }
            self.deleteExcludedProperties(commit.from);
            self.deleteExcludedProperties(commit.to);
            var patch = diff.diff(commit.to, commit.from);

            // If a patch edits a widget in an area that doesn't exist at all yet
            // in the receiving locale, create the area
            _.each(patch, function(value, key) {
              if (!draft[key]) {
                if (commit.from[key] && (commit.from[key].type === 'area')) {
                  draft[key] = {
                    type: 'area',
                    items: []
                  };
                }
              }
            });
            
            // Step 1: find all the sub-objects with an _id property that are
            // present in the docs and sort them in descending order by
            // depth. These will be schema array items and widgets

            var draftObjects = getObjects(draft);
            var fromObjects = getObjects(from);
            var toObjects = getObjects(to);
            
            function getObjects(doc) {
              var objects = [];
              var dotPaths = {};
              var dots = {};
              self.apos.docs.walk(draft, function(doc, key, value, dotPath, ancestors) {
                if (value && (typeof(value) === 'object') && value._id) {
                  objects.push(value);
                  dotPaths[value._id] = dotPath;
                  dots[value._id] = 0;
                  for (var i = 0; (i < dotPath.length); i++) {
                    if (dotPath.charAt(i) === '.') {
                      dots[value._id]++;
                    }
                  }
                }
              });
                         
              objects.sort(function(a, b) {
                if (dots[a._id] > dots[b._id]) {
                  return -1;
                } else if (dots[a._id] > dots[b._id]) {
                  return 1;
                } else {
                  return 0;
                }
              });

              return {
                objects: objects,
                dotPaths: dotPaths,
                dots: dots,
                byId: _.indexBy(objects, '_id')
              };

            }
            
            // Step 2: iterate over those objects, patching directly as appropriate
            
            _.each(toObjects.objects, function(value) {
              // Deleted
              if (!_.has(fromObjects.dotPaths[value._id])) {
                if (draftObjects.dotPaths[value._id]) {
                  deleteObject(draft, draftObjects, value._id);
                }
                return;
              }
            });
            _.each(fromObjects.objects, function(value) {
              // Modified. Could also be moved, so don't return
              if (!_.isEqual(value, fromObjects.byId[value._id])) {
                updateObject(draft, draftObjects, value._id);
              }
              // Moved. Look at neighbor ids, not indexes, to account for
              // existing divergences
              var toDotPath = toObjects.dotPaths[value._id];
              var fromDotPath = fromObjects.dotPaths[value._id];
              if (toDotPath !== fromDotPath) {
                var fromIndex = parseInt(_.last(fromDotPath.split('.')));
                var afterId;
                if (fromIndex > 0) {
                  for (var i = fromIndex - 1; (i >= 0); i++) {
                    var subPath = fromDotPath.split('.');
                    subPath.pop();
                    subPath.concat(i);
                    var obj = getObject(from, fromObjects, subPath.join('.'));
                    if (obj) {
                      var afterId = obj._id;
                      if (_.has(draftObjects.byId, afterId)) {
                        deleteObject(draft, draftObjects, value);
                        insertObjectAfter(draft, draftObjects, afterId, value);
                        return;
                      }
                    }
                  }  
                }
                deleteObject(draft, draftObjects, value);
                appendObject(draft, draftObjects, fromDotPath.replace(/\.\d$/, ''), value);
              }
            });

            // Step 3: remove all of these objects so jsondiffpatch doesn't consider them
            purgeObjects(draft, draftObjects);
            purgeObjects(commit.from, fromObjects);
            purgeObjects(commit.to, toObjects);
            
            // Step 4: patch as normal for everything that doesn't have an _id
            
            try {
              console.log('patch is: ', JSON.stringify(patch, null, '  '));
              console.log('draft is: ', JSON.stringify(draft, null, '  '));
              diff.patch(draft, patch);
            } catch (e) {
              errors.push(locale.replace(/\-draft$/, ''));
              console.error(e);
            }
            return callback(null);
            
            function getObject(context, objects, dotPath) {
              return deep(context, dotPath);
            }
            
            function deleteObject(context, objects, value) {
              var dotPath = objects.dotPaths[value._id];
              if (!dotPath) {
                return;
              }
              deep.set(context, dotPath, undefined);
            }
            
            function appendObject(context, objects, path, object) {
              var array = deep.get(context, path);
              if (!Array.isArray(array)) {
                return;
              }
              array.push(object);
            }
            
            function purgeObjects(context, objects) {
              _.each(objects, function(object) {
                deleteObject(context, objects, object);
              });
            }
            
          }
          // Resolve relationship ids back to the locale the draft is coming from
          function resolveToDestination(callback) {
            return self.resolveRelationships(req, draft, draft.workflowLocale, callback);
          }
          function update(callback) {
            return self.apos.docs.update(req, draft, callback); 
          }
        }, callback);
      }
    });
    
    self.deleteExcludedProperties = function(doc) {
      _.each(doc, function(val, key) {
        if (!self.includeProperty(key)) {
          delete doc[key];
        }
{'fraction_non_alphanumeric': 0.11178765569140861, 'fraction_numerical': 0.001044744445714099, 'mean_word_length': 1.8453320947515095, 'pattern_counts': {'":': 0, '<': 4, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 2, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1228844', 'n_tokens_mistral': 16761, 'n_tokens_neox': 16303, 'n_words': 4544}