<commit_before>var moment = require('moment');
var async = require('async');
var _ = require('@sailshq/lodash');
var fs = require('fs');
var url = require('url');

var defaultLocale = 'default';

module.exports = {

  // Cache sitemaps for 1 hour by default. Depending on pagerank
  // Google may look at your sitemap somewhere between daily and
  // monthly, so don't get your hopes up too far about changing this

  cacheLifetime: 60 * 60,

  piecesPerBatch: 100,

  moogBundle: {
    modules: [ 'apostrophe-site-map-custom-pages', 'apostrophe-site-map-pieces' ],
    directory: 'lib/modules'
  },

  afterConstruct: function(self) {
    self.apos.tasks.add(self.__meta.name, 'map', self.mapTask);
    self.apos.tasks.add(self.__meta.name, 'clear', self.clearTask);
    self.addRoutes();
    self.enableCache();
  },

  construct: function(self, options) {

    self.caching = true;

    self.cacheLifetime = options.cacheLifetime;

    self.piecesPerBatch = options.piecesPerBatch;

    self.clearTask = function(apos, argv, callback) {
      // Just forget the current sitemaps to make room
      // for regeneration on the next request
      return self.cache.clear(callback);
    };

    self.mapTask = function(apos, argv, callback) {
      if (argv['update-cache']) {
        self.caching = true;
      } else {
        self.caching = false;
      }

      if (!apos.options.baseUrl) {
        return callback(new Error(
          'You must specify the top-level baseUrl option when configuring Apostrophe\n' +
          'to use this task. Example: baseUrl: "http://mycompany.com"\n\n' +
          'Note there is NO TRAILING SLASH.\n\n' +
          'Usually you will only do this in data/local.js, on production.'
        ));
      }

      return self.map(callback);
    };

    self.map = function(callback) {

      self.workflow = self.apos.modules['apostrophe-workflow'];

      var argv = self.apos.argv;

      if (argv['exclude-types']) {
        self.excludeTypes = excludeTypes.concat(argv['exclude-types'].split(','));
      }

      if (self.caching) {
        self.cacheOutput = [];
      }
      return async.series([
        lock,
        init,
        map,
        hreflang,
        write,
        unlock
      ], callback);

      function lock(callback) {
        return self.apos.locks.lock('apostrophe-site-map', callback);
      }

      function init(callback) {
        self.format = argv.format || options.format || 'xml';

        self.indent = (typeof(argv.indent) !== 'undefined') ? argv.indent : options.indent;
        self.excludeTypes = options.excludeTypes || [];
        self.perLocale = options.perLocale || argv['per-locale'];
        // Exception: plaintext sitemaps and sitemap indexes don't go
        // together, so we can presume that if they explicitly ask
        // for plaintext they are just doing content strategy and we
        // should produce a single report
        if (self.format === 'text') {
          self.perLocale = false;
        }
        return callback(null);
      }

      function map(callback) {
        self.maps = {};
        self.today = moment().format('YYYY-MM-DD');

        var locales = [ defaultLocale ];

        if (self.workflow) {
          locales = _.filter(_.keys(self.workflow.locales), function(locale) {
            return !locale.match(/-draft$/) && !self.workflow.locales[locale].private;
          });
        }

        return async.eachSeries(locales, function(locale, callback) {
          var req = self.apos.tasks.getAnonReq({ locale: locale });
          req.locale = locale;
          return async.series([
            _.partial(self.getPages, req, locale),
            _.partial(self.getPieces, req, locale),
            function(callback) {
              if (self.custom.length === 1) {
                return self.custom(callback);
              } else {
                return self.custom(req, locale, callback);
              }
            }
          ], callback);
        }, function(err) {
          if (err) {
            return callback(err);
          }
          return callback(null);
        });
      }

      function hreflang(callback) {

        var alternativesByGuid = {};

        each(function(entry) {
          if (!alternativesByGuid[entry.url.workflowGuid]) {
            alternativesByGuid[entry.url.workflowGuid] = [];
          }
          alternativesByGuid[entry.url.workflowGuid].push(entry);
        });

        each(function(entry) {
          entry.url['xhtml:link'] = [];
          var alternatives = alternativesByGuid[entry.url.workflowGuid];
          _.each(alternatives, function(alternative) {
            if (alternative === entry) {
              return;
            }
            entry.url['xhtml:link'].push({
              _attributes: {
                rel: 'alternate',
                hreflang: alternative.url.workflowLocale,
                href: alternative.url.loc
              }
            });
          });
        });

        each(function(entry) {
          delete entry.url.workflowLocale;
          delete entry.url.workflowGuid;
        });

        return setImmediate(callback);

        function each(iterator) {
          _.each(self.maps, function(map) {
            _.each(map, function(entry) {
              if (typeof(entry) !== 'object') {
                return;
              }
              if (!entry.url.workflowGuid) {
                return;
              }
              iterator(entry);
            });
          });
        }

      }

      function write(callback) {
        return self.writeSitemap(callback);
      }

      function unlock(callback) {
        return self.apos.locks.unlock('apostrophe-site-map', callback);
      }
    };

    self.getPages = function(req, locale, callback) {
      return self.findPages(req).toObject(function(err, home) {
        if (err) {
          return callback(err);
        }
        if (!home) {<commit_msg>Ignores workflow needs for deleting props<commit_after>var moment = require('moment');
var async = require('async');
var _ = require('@sailshq/lodash');
var fs = require('fs');
var url = require('url');

var defaultLocale = 'default';

module.exports = {

  // Cache sitemaps for 1 hour by default. Depending on pagerank
  // Google may look at your sitemap somewhere between daily and
  // monthly, so don't get your hopes up too far about changing this

  cacheLifetime: 60 * 60,

  piecesPerBatch: 100,

  moogBundle: {
    modules: [ 'apostrophe-site-map-custom-pages', 'apostrophe-site-map-pieces' ],
    directory: 'lib/modules'
  },

  afterConstruct: function(self) {
    self.apos.tasks.add(self.__meta.name, 'map', self.mapTask);
    self.apos.tasks.add(self.__meta.name, 'clear', self.clearTask);
    self.addRoutes();
    self.enableCache();
  },

  construct: function(self, options) {

    self.caching = true;

    self.cacheLifetime = options.cacheLifetime;

    self.piecesPerBatch = options.piecesPerBatch;

    self.clearTask = function(apos, argv, callback) {
      // Just forget the current sitemaps to make room
      // for regeneration on the next request
      return self.cache.clear(callback);
    };

    self.mapTask = function(apos, argv, callback) {
      if (argv['update-cache']) {
        self.caching = true;
      } else {
        self.caching = false;
      }

      if (!apos.options.baseUrl) {
        return callback(new Error(
          'You must specify the top-level baseUrl option when configuring Apostrophe\n' +
          'to use this task. Example: baseUrl: "http://mycompany.com"\n\n' +
          'Note there is NO TRAILING SLASH.\n\n' +
          'Usually you will only do this in data/local.js, on production.'
        ));
      }

      return self.map(callback);
    };

    self.map = function(callback) {

      self.workflow = self.apos.modules['apostrophe-workflow'];

      var argv = self.apos.argv;

      if (argv['exclude-types']) {
        self.excludeTypes = excludeTypes.concat(argv['exclude-types'].split(','));
      }

      if (self.caching) {
        self.cacheOutput = [];
      }
      return async.series([
        lock,
        init,
        map,
        hreflang,
        write,
        unlock
      ], callback);

      function lock(callback) {
        return self.apos.locks.lock('apostrophe-site-map', callback);
      }

      function init(callback) {
        self.format = argv.format || options.format || 'xml';

        self.indent = (typeof(argv.indent) !== 'undefined') ? argv.indent : options.indent;
        self.excludeTypes = options.excludeTypes || [];
        self.perLocale = options.perLocale || argv['per-locale'];
        // Exception: plaintext sitemaps and sitemap indexes don't go
        // together, so we can presume that if they explicitly ask
        // for plaintext they are just doing content strategy and we
        // should produce a single report
        if (self.format === 'text') {
          self.perLocale = false;
        }
        return callback(null);
      }

      function map(callback) {
        self.maps = {};
        self.today = moment().format('YYYY-MM-DD');

        var locales = [ defaultLocale ];

        if (self.workflow) {
          locales = _.filter(_.keys(self.workflow.locales), function(locale) {
            return !locale.match(/-draft$/) && !self.workflow.locales[locale].private;
          });
        }

        return async.eachSeries(locales, function(locale, callback) {
          var req = self.apos.tasks.getAnonReq({ locale: locale });
          req.locale = locale;
          return async.series([
            _.partial(self.getPages, req, locale),
            _.partial(self.getPieces, req, locale),
            function(callback) {
              if (self.custom.length === 1) {
                return self.custom(callback);
              } else {
                return self.custom(req, locale, callback);
              }
            }
          ], callback);
        }, function(err) {
          if (err) {
            return callback(err);
          }
          return callback(null);
        });
      }

      function hreflang(callback) {

        var alternativesByGuid = {};

        each(function(entry) {
          if (!alternativesByGuid[entry.url.workflowGuid]) {
            alternativesByGuid[entry.url.workflowGuid] = [];
          }
          alternativesByGuid[entry.url.workflowGuid].push(entry);
        });

        each(function(entry) {
          entry.url['xhtml:link'] = [];
          var alternatives = alternativesByGuid[entry.url.workflowGuid];
          _.each(alternatives, function(alternative) {
            if (alternative === entry) {
              return;
            }
            entry.url['xhtml:link'].push({
              _attributes: {
                rel: 'alternate',
                hreflang: alternative.url.workflowLocale,
                href: alternative.url.loc
              }
            });
          });
        });

        each(function(entry) {
          delete entry.url.workflowLocale;
          delete entry.url.workflowGuid;
        }, true);

        return setImmediate(callback);

        function each(iterator, ignoreWorkflow) {
          _.each(self.maps, function(map) {
            _.each(map, function(entry) {
              if (typeof(entry) !== 'object') {
                return;
              }

              if (!entry.url.workflowGuid && !ignoreWorkflow) {
                return;
              }
              iterator(entry);
            });
          });
        }

      }

      function write(callback) {
        return self.writeSitemap(callback);
      }

      function unlock(callback) {
        return self.apos.locks.unlock('apostrophe-site-map', callback);
      }
    };

    self.getPages = function(req, locale, callback) {
      return self.findPages(req).toObject(function(err, home) {
        if (err) {
          return callback(err);
        }
        if (!home) {
{'fraction_non_alphanumeric': 0.12635892289680548, 'fraction_numerical': 0.0015052684395383843, 'mean_word_length': 1.9118578037496956, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '689544', 'n_tokens_mistral': 3443, 'n_tokens_neox': 3299, 'n_words': 728}