<commit_before>const blc = require('broken-link-checker');
const targets = require('../targets.json');

const options = {};

/**
 * Monitoring
 */

exports.run = runChecks;

function runChecks() {
  console.log('running checks...');
  const tally = {
    websites: {
      complete: [],
      broken: [],
      excluded: [],
      completeCount: 0,
      brokenCount: 0,
      excludedCount: 0,
    },
  };

  /**
   * Scan websites
   */
  const siteChecker = new blc.SiteChecker(options, {
    // robots: (robots, data) => {},
    // html: (tree, robots) => {},
    // junk: (result) => {},
    link: (result) => {
      const url = result.url.original;
      const resolved = result.url.resolved;
      const base = result.base.original;
      console.log(`${base}: ${resolved}: ${result.http.cached ? 'cached' : 'not in cache'}`);

      // Handle completion
      if (!result.http.cached) {
        tally.websites.complete.push(url);
        tally.websites.completeCount += 1;

        if (result.broken) {
          const reason = result.brokenReason;
          tally.websites.broken.push({ url, reason });
          tally.websites.brokenCount += 1;
        }

        if (result.excluded) {
          const reason = result.excludedReason;
          tally.websites.excluded.push({ url, reason });
          tally.websites.excludedCount += 1;
        }
      }
    },
    // page: (err, url, data) => {},
    // site: (err, url, data) => {},
    end: () => {
      console.log('Website check complete');
      console.log(tally.websites);
    },
  });

  targets.websites.forEach((website) => {
    siteChecker.enqueue(website);
  });

  /**
   * Scan Services/APIs
   */
}<commit_msg>Services Check<commit_after>const blc = require('broken-link-checker');
const targets = require('../targets.json');

const options = {};

/**
 * Monitoring
 */

exports.run = runChecks;

function runChecks() {
  console.log('running checks...');
  const tally = {
    websites: {
      complete: [],
      broken: [],
      excluded: [],
      completeCount: 0,
      brokenCount: 0,
      excludedCount: 0,
    },
    services: {
      complete: [],
      broken: [],
      excluded: [],
      completeCount: 0,
      brokenCount: 0,
      excludedCount: 0,
    },
  };

  /**
   * Scan websites
   */
  const siteChecker = new blc.SiteChecker(options, {
    // robots: (robots, data) => {},
    // html: (tree, robots) => {},
    // junk: (result) => {},
    link: (result) => {
      const resolved = result.url.resolved;
      const base = result.base.original;
      const statusCode = result.http.response.statusCode;

      // Handle completion
      if (!result.http.cached) {
        tally.websites.complete.push({ base, resolved, statusCode });
        tally.websites.completeCount += 1;

        if (result.broken) {
          const reason = result.brokenReason;
          tally.websites.broken.push({ base, resolved, reason });
          tally.websites.brokenCount += 1;
        }

        if (result.excluded) {
          const reason = result.excludedReason;
          tally.websites.excluded.push({ base, resolved, reason });
          tally.websites.excludedCount += 1;
        }
      }
    },
    // page: (err, url, data) => {},
    // site: (err, url, data) => {},
    end: () => {
      console.log('Website check complete');
      console.log(tally.websites);
    },
  });

  targets.websites.forEach((website) => {
    siteChecker.enqueue(website);
  });

  /**
   * Scan Services/APIs
   */
  const urlChecker = new blc.UrlChecker(options, {
    link: (result) => {
      const url = result.url.original;
      const resolved = result.url.resolved;
      const statusCode = result.http.response.statusCode;

      // Handle completion
      tally.services.complete.push({ url, resolved, statusCode });
      tally.services.completeCount += 1;

      if (result.broken) {
        const reason = result.brokenReason;
        tally.services.broken.push({ url, reason });
        tally.services.brokenCount += 1;
      }

      if (result.excluded) {
        const reason = result.excludedReason;
        tally.services.excluded.push({ url, reason });
        tally.services.excludedCount += 1;
      }
    },
    end: () => {
      console.log('Service/API check complete');
      console.log(tally.services);
    },
  });

  targets.services.forEach((service) => {
    urlChecker.enqueue(service);
  });
}
{'fraction_non_alphanumeric': 0.16187457550373557, 'fraction_numerical': 0.004075164138555581, 'mean_word_length': 2.217771303714494, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 22, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7340749', 'n_tokens_mistral': 1418, 'n_tokens_neox': 1349, 'n_words': 265}