<commit_before>/**
 * sitemap-index-urls.js
 * 
 * Generates the urls for a sitemap-index.
 * 
 * Copyright (c) 2013 - 2022 <NAME>, LocalNerve, contributors
 * Licensed under the MIT license.
 */
const _ = require("lodash");
const smc = require("./sitemap-collection");

function sitemapIndexUrls (smLib, options, parseResult) {
  const sitemapUrls = [];
  const sitemapIndexOptions = Object.assign({}, options, {
    outputPath: undefined
  });

  // Check if we should process each sitemap in the index.
  _.forEach(
    // if the sitemap index is malformed, just blow up
    parseResult.sitemapindex.sitemap,
    sitemapNode => {
      // optionally ignore current sitemaps by sitemap policy
      const shouldProcess = !options.sitemapPolicy ||
        !smLib.stillCurrent(sitemapNode, sitemapIndexOptions);

      if (shouldProcess) {
        // if sitemap index is malformed, just blow up
        sitemapUrls.push(sitemapNode.loc[0]);
      }
    }
  );

  return {
    sitemapUrls,
    sitemapIndexOptions
  };
}

function sitemapIndex (smLib, options, parseResult) {
  const {
    sitemapUrls,
    sitemapIndexOptions
  } = sitemapIndexUrls(smLib, options, parseResult);<commit_msg>update syntax, add missing jsdoc<commit_after>/**
 * sitemap-index.js
 * 
 * Generates the urls for a sitemap-index.
 * 
 * Copyright (c) 2013 - 2022 <NAME>, LocalNerve, contributors
 * Licensed under the MIT license.
 */
const _ = require("lodash");
const smc = require("./sitemap-collection");

/**
 * Create a collection of sitemap urls to process by policy, form sitemapIndexOptions.
 *
 * @param {Object} smLib - Sitemap module
 * @param {Object} options - sitemap index options
 * @param {Boolean} options.sitemapPolicy - True if sitemap policy should be used.
 * @param {Object} parseResult - xml2js sax parse result object
 * @returns {Object} The sitemapUrls collection and sitemapIndexOptions
 */
function sitemapIndexUrls (smLib, options, parseResult) {
  const sitemapUrls = [];
  const sitemapIndexOptions = {
    ...options,
    ...{ outputPath: undefined }
  };

  // Check if we should process each sitemap in the index.
  _.forEach(
    // if the sitemap index is malformed, just blow up
    parseResult.sitemapindex.sitemap,
    sitemapNode => {
      // optionally ignore current sitemaps by sitemap policy
      const shouldProcess = !options.sitemapPolicy ||
        !smLib.stillCurrent(sitemapNode, sitemapIndexOptions);

      if (shouldProcess) {
        // if sitemap index is malformed, just blow up
        sitemapUrls.push(sitemapNode.loc[0]);
      }
    }
  );

  return {
    sitemapUrls,
    sitemapIndexOptions
  };
}

/**
 * Collect sitemap urls from a sitemap index file for processing.
 *
 * @param {Object} smLib - sitemap module
 * @param {Object} options - sitemapIndex options
 * @param {Object} parseResult - xml2js sax parse result object
 * @returns {Promise} resolves when all sitemaps are processed, reject as soon as one fails
 */
function sitemapIndex (smLib, options, parseResult) {
  const {
    sitemapUrls,
    sitemapIndexOptions
  } = sitemapIndexUrls(smLib, options, parseResult);
{'fraction_non_alphanumeric': 0.10147247119078105, 'fraction_numerical': 0.006402048655569782, 'mean_word_length': 3.3645251396648046, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4022802', 'n_tokens_mistral': 1012, 'n_tokens_neox': 980, 'n_words': 286}