<commit_before>'use strict';

const cheerio = require('cheerio');
const Promise = require('bluebird');
const minimatch = require('minimatch');

function concatJS(data) {
  const hexo = this;
  const options = hexo.config.js_concator;
  // Reture if disabled.
  if (options.enable === false) return;

  const log = hexo.log || console;
  const route = hexo.route;
  const routeList = route.list();

  let include = options.include;
  if (include && !Array.isArray(include)) include = [include];

  // 1. get all local scripts
  const htmls = {};
  const bundlePath = options.bundle_path;
  return Promise.all(routeList.filter(path => path.endsWith('.html')).map(path => {
    // 1. get the htmls that has local scripts
    return new Promise((resolve, reject) => {
      const html = route.get(path);
      let htmlTxt = ''
      html.on('data', (chunk) => (htmlTxt += chunk));
      html.on('end', () => {
        const $ = cheerio.load(htmlTxt);
        const $scripts = {};
        const srcs = [];
        $('script[src]').each((idx, ele) => {
          const $script = $(ele);
          let src = $script.attr('src');
          if (src && src.length) {
            // local script
            if (!src.startsWith('//') && !src.startsWith('http')) {
              src = route.format(src);
              srcs.push(src);
              $scripts[src] = $script;
            }
          }
        });
        if (srcs.length) {
          htmls[path] = { path, $, srcs, $scripts };
        }
        resolve();
      });
    });
  }))
    // 2. extract common scripts
    .then(() => {
      const htmlPaths = Object.keys(htmls);
      const commons = [];
      // collect the scripts
      for (const path of htmlPaths) {
        const html = htmls[path];
        for (const src of html.srcs) {
          if (commons.indexOf(src) === -1) {
            // if a script exists in every html which has scripts,
            // or match the pattern in include array
            if (htmlPaths.every(path => htmls[path].srcs.indexOf(src) !== -1) ||
              include.some(pattern => minimatch(src, pattern, { matchBase: true }))) {
                // remove
                log.log('Remove %s from %s', src, path);
                html.$scripts[src].remove()
                delete html.$scripts[src];
                html.srcs.splice(html.srcs.indexOf(src), 1);
                commons.push(src);
            }
          } else {
            // remove
            log.log('Remove %s from %s', src, path);
            html.$scripts[src].remove()
            delete html.$scripts[src];
            html.srcs.splice(html.srcs.indexOf(src), 1);
          }
        }
      }

      if (commons.length > 1) {
        // add the bundle script to all html
        for (const path of htmlPaths) {
          const html = htmls[path];
          html.$('body').append(`<script type="text/javascript" src="${route.format(bundlePath)}"></script>`);
          route.set(path, html.$.html());
          log.log('Add %s to %s', route.format(bundlePath), path);
        }
      }
      return commons;
    })
    // 3. concat the script
    .then((scripts) => {
      if (scripts.length < 1) return;
      log.log('Try to concat %s scripts', scripts.length);
      return Promise.all(scripts.map(path => {
        return new Promise((resolve, reject) => {
          const script = route.get(path);
          let scriptTxt = '';
          script.on('data', chunk => (scriptTxt += chunk));
          script.on('end', () => {
            route.remove(path);
            log.info('concat script %s', path);
            resolve(scriptTxt);
          });
        });
      })).then(results => {
        const bundleScript = results.reduce((txt, script) => (txt += script), '');
        route.set(route.format(bundlePath), bundleScript);
        log.info('finish concat js script');
      });
    });
}

module.exports = concatJS;<commit_msg>fix: format the log info<commit_after>'use strict';

const cheerio = require('cheerio');
const Promise = require('bluebird');
const minimatch = require('minimatch');

function concatJS(data) {
  const hexo = this;
  const options = hexo.config.js_concator;
  // Reture if disabled.
  if (options.enable === false) return;

  const log = hexo.log || console;
  const route = hexo.route;
  const routeList = route.list();

  let include = options.include;
  if (include && !Array.isArray(include)) include = [include];

  // 1. get all local scripts
  const htmls = {};
  const bundlePath = options.bundle_path;
  return Promise.all(routeList.filter(path => path.endsWith('.html')).map(path => {
    // 1. get the htmls that has local scripts
    return new Promise((resolve, reject) => {
      const html = route.get(path);
      let htmlTxt = ''
      html.on('data', (chunk) => (htmlTxt += chunk));
      html.on('end', () => {
        const $ = cheerio.load(htmlTxt);
        const $scripts = {};
        const srcs = [];
        $('script[src]').each((idx, ele) => {
          const $script = $(ele);
          let src = $script.attr('src');
          if (src && src.length) {
            // local script
            if (!src.startsWith('//') && !src.startsWith('http')) {
              src = route.format(src);
              srcs.push(src);
              $scripts[src] = $script;
            }
          }
        });
        if (srcs.length) {
          htmls[path] = { path, $, srcs, $scripts };
        }
        resolve();
      });
    });
  }))
    // 2. extract common scripts
    .then(() => {
      const htmlPaths = Object.keys(htmls);
      const commons = [];
      // collect the scripts
      for (const path of htmlPaths) {
        const html = htmls[path];
        for (const src of html.srcs) {
          if (commons.indexOf(src) === -1) {
            // if a script exists in every html which has scripts,
            // or match the pattern in include array
            if (htmlPaths.every(path => htmls[path].srcs.indexOf(src) !== -1) ||
              include.some(pattern => minimatch(src, pattern, { matchBase: true }))) {
                // remove
                log.log('update Concate JS: remove %s from %s', src, path);
                html.$scripts[src].remove()
                delete html.$scripts[src];
                html.srcs.splice(html.srcs.indexOf(src), 1);
                commons.push(src);
            }
          } else {
            // remove
            log.log('update Concate JS: remove %s from %s', src, path);
            html.$scripts[src].remove()
            delete html.$scripts[src];
            html.srcs.splice(html.srcs.indexOf(src), 1);
          }
        }
      }

      if (commons.length > 1) {
        // add the bundle script to all html
        for (const path of htmlPaths) {
          const html = htmls[path];
          html.$('body').append(`<script type="text/javascript" src="${route.format(bundlePath)}"></script>`);
          route.set(path, html.$.html());
          log.log('update Concate JS: add %s to %s', route.format(bundlePath), path);
        }
      }
      return commons;
    })
    // 3. concat the script
    .then((scripts) => {
      if (scripts.length < 1) return;
      log.log('update Concate JS: try to concat %s scripts', scripts.length);
      return Promise.all(scripts.map(path => {
        return new Promise((resolve, reject) => {
          const script = route.get(path);
          let scriptTxt = '';
          script.on('data', chunk => (scriptTxt += chunk));
          script.on('end', () => {
            route.remove(path);
            log.info('update Concate JS: concat script %s', path);
            resolve(scriptTxt);
          });
        });
      })).then(results => {
        const bundleScript = results.reduce((txt, script) => (txt += script), '');
        route.set(route.format(bundlePath), bundleScript);
        log.info('update Concate JS: finish concat js script');
      });
    });
}

module.exports = concatJS;
{'fraction_non_alphanumeric': 0.15935102502829832, 'fraction_numerical': 0.002515406867060747, 'mean_word_length': 1.9246046340566385, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 41, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2433276', 'n_tokens_mistral': 2355, 'n_tokens_neox': 2321, 'n_words': 575}