<commit_before>#!/usr/bin/env node

const color   = require('cli-color');
const Sitemap = require('sitemap');
const program = require('commander');
const fs      = require('fs');
const Path    = require('path');
const common  = require('./common');
const urls    = require('./config/sitemap_urls');

program
    .version('0.2.0')
    .description('Generate sitemap.xml')
    .parse(process.argv);

const url_prefix = 'https://www.binary.com/';
const filename   = 'sitemap.xml';
let excluded     = 0;

const createSitemap = () => {
    const sitemap = Sitemap.createSitemap({
        hostname : url_prefix,
        cacheTime: 600000,
    });

    common.languages
        .map(lang => lang.toLowerCase())
        .forEach((lang) => {
            urls.forEach((entry) => {
                if (!common.isExcluded(entry[3], lang)) {
                    sitemap.add({
                        url       : `${url_prefix}${lang}/${entry[0]}.html`,
                        changefreq: entry[1],
                        priority  : entry[2],
                    });
                } else {
                    excluded++;
                }
            });
        });

    fs.writeFileSync(Path.join(common.root_path, filename), sitemap.toString());
};

const start = Date.now();
process.stdout.write(common.messageStart(`Generating ${filename}`));
createSitemap();
process.stdout.write(common.messageEnd(Date.now() - start));

// Report details
const langs_count = common.languages.length;
const total_count = langs_count * urls.length - excluded;
console.log(`  ${color.green(total_count)} URL nodes total (${color.cyan(langs_count)} Languages ${color.yellowBright('*')} ${color.cyan(urls.length)} URLs ${color.yellowBright('-')} ${color.cyan(excluded)} Excluded)`); // eslint-disable-line no-console<commit_msg>Split sitemap based on the given configuration<commit_after>#!/usr/bin/env node

const color   = require('cli-color');
const Sitemap = require('sitemap');
const program = require('commander');
const fs      = require('fs');
const Path    = require('path');
const common  = require('./common');
const urls    = require('./config/sitemap_urls');

program
    .version('0.2.0')
    .description('Generate sitemap.xml')
    .parse(process.argv);

const config = [
    {
        url_prefix : 'https://www.binary.com/',
        filename   : 'sitemap.xml',
        lang_filter: '^(?!id$)',
    },
    {
        url_prefix : 'https://www.binary.me/',
        filename   : 'sitemap.id.xml',
        lang_filter: '^id$',
    },
];
let excluded;

const getApplicableLanguages = (lang_filter) => common.languages.filter(lang => new RegExp(lang_filter, 'i').test(lang));

const createSitemap = (conf) => {
    excluded = 0;

    const sitemap = Sitemap.createSitemap({
        hostname : conf.url_prefix,
        cacheTime: 600000,
    });

    getApplicableLanguages(conf.lang_filter)
        .map(lang => lang.toLowerCase())
        .forEach((lang) => {
            urls.forEach((entry) => {
                if (!common.isExcluded(entry[3], lang)) {
                    sitemap.add({
                        url       : `${conf.url_prefix}${lang}/${entry[0]}.html`,
                        changefreq: entry[1],
                        priority  : entry[2],
                    });
                } else {
                    excluded++;
                }
            });
        });

    fs.writeFileSync(Path.join(common.root_path, conf.filename), sitemap.toString());
};

config.forEach((conf) => {
    const start = Date.now();
    process.stdout.write(common.messageStart(`Generating ${conf.filename}`));

    createSitemap(conf);

    process.stdout.write(common.messageEnd(Date.now() - start));

    // Report details
    const langs_count = getApplicableLanguages(conf.lang_filter).length;
    const total_count = langs_count * urls.length - excluded;
    console.log(`  ${color.green(total_count)} URL nodes total (${color.cyan(langs_count)} Languages ${color.yellowBright('*')} ${color.cyan(urls.length)} URLs ${color.yellowBright('-')} ${color.cyan(excluded)} Excluded)\n`); // eslint-disable-line no-console
});
{'fraction_non_alphanumeric': 0.1651287032540068, 'fraction_numerical': 0.006799417192812044, 'mean_word_length': 2.4936386768447836, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 14, 'https://': 3, 'lorem ipsum': 0, 'www.': 3, 'xml': 5}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2803653', 'n_tokens_mistral': 1289, 'n_tokens_neox': 1256, 'n_words': 221}