<commit_before>var request = require('request'), // For making HTTP requests
    cheerio = require('cheerio'), // For parsing the DOM
    events = require('events'), // For emitters and listeners
    chalk = require('chalk'), // For coloring cli outputs
    fs = require('fs'), // For writing to file
    URI = require('URIjs'); // For using URIs as objects

var SiteMapCrawler = function(url, protocol, timeout, outputFile) {
  this.eventEmitter = new events.EventEmitter();

  this.initialUrl = url;
  this.protocol = protocol;
  this.timeout = parseInt(timeout) || 3000; // 3 seconds
  this.retries = 5;

  this.uriList = [];
  this.uriQueue = [];
  this.retryList = [];
  this.invalidLinkCount = 0;

  this.domain = new URI(url).domain();
  console.log('Domain crawled is: ' + this.domain);

  // Initialize our listener
  this.printFinalUrlList(outputFile);
};

SiteMapCrawler.prototype.setInfoLevel = function(info) {
  this.info = info;
};

SiteMapCrawler.prototype.setRetries = function(retry) {
  this.retries = retry;
};

SiteMapCrawler.prototype.crawlUrl = function(url) {
  siteMapCrawler = this;
  var anchorList = [];
  var options = {
    method: 'GET',
    url: url,
    timeout: siteMapCrawler.timeout,
  };

  // Log what site we are crawling on -i option
  if(siteMapCrawler.info) {
    console.log('Crawling ' + url);
  // Otherwise mark our action (for console visual aid)
  } else { process.stdout.write('.'); }

  var req = request(options, function(err, res, body) {

    if (res) {
      if (res.statusCode >= 200 && res.statusCode < 300) { // Make sure we have a valid response
        if (body) {
          var $ = cheerio.load(body);  //Let's use JQuery selectors
          $('a').each( function(i, elem) { // Loop over anchor tags in our body
            var link = $(elem).attr('href'); // Extract the link

            // Make sure the link exists and is not already in our list
            if (link && anchorList.indexOf(link) === -1) {
              anchorList.push(link);
            }
          });

          // Get the uri from the origin of the response
          // This is useful if we were redirected
          var requestUri = res.request.uri.href;
          // Call our method to parse the anchors
          siteMapCrawler.updateUrlList(anchorList, requestUri);

        }
      }
      else if (res.statusCode >= 400) {
        console.log(chalk.bold.yellow('\nGET returned status code: %s for %s'), res.statusCode, url);
        siteMapCrawler.invalidLinkCount++;
      }
      siteMapCrawler.eventEmitter.emit('urlProccessed', url);
    }
    // Something went wrong with the request
    if (err) {
      // Retry if this was a read error on our part
      // This happens because we're pooling through our connections too fast
      if (err.code === 'ETIMEDOUT' || err.code === 'ESOCKETTIMEDOUT') {
        var retryList = siteMapCrawler.retryList;
        var retriedLink = {};
        var isFound = false;

        // console.log(chalk.cyan("ETIMEDOUT occured"));

        for(var i=0; i < retryList.length; i++){
          if(retryList[i].url === url) {
            isFound = true;
            retryList[i].try++;
            retriedLink = retryList[i];
          }
        }

        if(isFound === true) {
          if(retriedLink.try > siteMapCrawler.retries) {
            console.log(chalk.bold.yellow('\nRetries exceeded for %s'), url);
            siteMapCrawler.eventEmitter.emit('urlProccessed', url);
            return; // Because we don't want to start a new crawl
          }
        } else {
          // Make a retry link
          retriedLink = {
            url: url,
            try: 1
          };

          // Add link to list of retried url
          siteMapCrawler.retryList.push(retriedLink);
        }
        siteMapCrawler.crawlUrl(url);

      } else {
        // The url actually doesn't work, take it out of the queue
        console.log(chalk.red('\n' + err));
        siteMapCrawler.eventEmitter.emit('urlProccessed', url);
      }
    }
  });
};

SiteMapCrawler.prototype.updateUrlList = function(anchorList, originUrl) {
  siteMapCrawler = this;
  for(var i=0; i < anchorList.length; i++) {
    // Make new URIs from our anchor list
    var tempUri = URI(anchorList[i]);
    // Remove any trailing query strings
    tempUri = tempUri.search("");
    // Remove any trailing hashes
    tempUri = tempUri.fragment("");

    if (tempUri.is('relative')) { // Ex: Relative would be "/about"
      // Append our relative uri to our main one
      tempUri = URI(originUrl).pathname(tempUri.href());
    }

    // Normalize the subdomain if not specified
    if(tempUri.subdomain() === '') {
      tempUri = tempUri.subdomain('www');
    }

    // Normalize the protocol
    tempUri = tempUri.protocol(siteMapCrawler.protocol);

    // Remove irregularities in URL (like capital letters)
    tempUri = tempUri.normalize();<commit_msg>Added image endpoints<commit_after>var request = require('request'), // For making HTTP requests
    cheerio = require('cheerio'), // For parsing the DOM
    events = require('events'), // For emitters and listeners
    chalk = require('chalk'), // For coloring cli outputs
    fs = require('fs'), // For writing to file
    URI = require('URIjs'); // For using URIs as objects

var SiteMapCrawler = function(url, protocol, timeout, outputFile) {
  this.eventEmitter = new events.EventEmitter();

  this.initialUrl = url;
  this.protocol = protocol;
  this.timeout = parseInt(timeout) || 3000; // 3 seconds
  this.retries = 5;

  this.uriList = [];
  this.uriQueue = [];
  this.retryList = [];
  this.invalidLinkCount = 0;

  this.domain = new URI(url).domain();
  console.log('Domain crawled is: ' + this.domain);

  // Initialize our listener
  this.printFinalUrlList(outputFile);
};

SiteMapCrawler.prototype.setInfoLevel = function(info) {
  this.info = info;
};

SiteMapCrawler.prototype.setRetries = function(retry) {
  this.retries = retry;
};

SiteMapCrawler.prototype.crawlUrl = function(url) {
  siteMapCrawler = this;
  var anchorList = [];
  var options = {
    method: 'GET',
    url: url,
    timeout: siteMapCrawler.timeout,
  };

  // Log what site we are crawling on -i option
  if(siteMapCrawler.info) {
    console.log('Crawling ' + url);
  // Otherwise mark our action (for console visual aid)
  } else { process.stdout.write('.'); }

  var req = request(options, function(err, res, body) {

    if (res) {
      if (res.statusCode >= 200 && res.statusCode < 300) { // Make sure we have a valid response
        if (body) {
          var $ = cheerio.load(body);  //Let's use JQuery selectors
          $('a').each( function(i, elem) { // Loop over anchor tags in our body
            var link = $(elem).attr('href'); // Extract the link

            // Make sure the link exists and is not already in our list
            if (link && anchorList.indexOf(link) === -1) {
              anchorList.push(link);
            }
          });

          // Do the same for all image links
          $('img').each( function(i, elem) {
            var img = $(elem).attr('src');
            var imgList = [];
            // Make sure the link exists and is not already in our list
            if (img && imgList.indexOf(img) === -1) {
              anchorList.push(img);
            }
          });

          // Find static css and js files the same way

          // Get the uri from the origin of the response
          // This is useful if we were redirected
          var requestUri = res.request.uri.href;
          // Call our method to parse the anchors
          siteMapCrawler.updateUrlList(anchorList, requestUri);

        }
      }
      else if (res.statusCode >= 400) {
        console.log(chalk.bold.yellow('\nGET returned status code: %s for %s'), res.statusCode, url);
        siteMapCrawler.invalidLinkCount++;
      }
      siteMapCrawler.eventEmitter.emit('urlProccessed', url);
    }
    // Something went wrong with the request
    if (err) {
      // Retry if this was a read error on our part
      // This happens because we're pooling through our connections too fast
      if (err.code === 'ETIMEDOUT' || err.code === 'ESOCKETTIMEDOUT') {
        var retryList = siteMapCrawler.retryList;
        var retriedLink = {};
        var isFound = false;

        // console.log(chalk.cyan("ETIMEDOUT occured"));

        for(var i=0; i < retryList.length; i++){
          if(retryList[i].url === url) {
            isFound = true;
            retryList[i].try++;
            retriedLink = retryList[i];
          }
        }

        if(isFound === true) {
          if(retriedLink.try > siteMapCrawler.retries) {
            console.log(chalk.bold.yellow('\nRetries exceeded for %s'), url);
            siteMapCrawler.eventEmitter.emit('urlProccessed', url);
            return; // Because we don't want to start a new crawl
          }
        } else {
          // Make a retry link
          retriedLink = {
            url: url,
            try: 1
          };

          // Add link to list of retried url
          siteMapCrawler.retryList.push(retriedLink);
        }
        siteMapCrawler.crawlUrl(url);

      } else {
        // The url actually doesn't work, take it out of the queue
        console.log(chalk.red('\n' + err));
        siteMapCrawler.eventEmitter.emit('urlProccessed', url);
      }
    }
  });
};

SiteMapCrawler.prototype.updateUrlList = function(anchorList, originUrl) {
  siteMapCrawler = this;
  for(var i=0; i < anchorList.length; i++) {
    // Make new URIs from our anchor list
    var tempUri = URI(anchorList[i]);
    // Remove any trailing query strings
    tempUri = tempUri.search("");
    // Remove any trailing hashes
    tempUri = tempUri.fragment("");

    if (tempUri.is('relative')) { // Ex: Relative would be "/about"
      // Append our relative uri to our main one
      var encodedPath = URI(tempUri.href()).pathname(true);
      tempUri = URI(originUrl).pathname(encodedPath);
    }

    // Normalize the subdomain if not specified
    if(tempUri.subdomain() === '') {
      tempUri = tempUri.subdomain('www');
    }

    // Normalize the protocol
    tempUri = tempUri.protocol(siteMapCrawler.protocol);

    // Remove irregularities in URL (like capital letters)
    tempUri = tempUri.normalize();
    //tempUri = tempUri.encode();
{'fraction_non_alphanumeric': 0.1265614408831219, 'fraction_numerical': 0.0039701752687130824, 'mean_word_length': 2.3145057766367136, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1828285', 'n_tokens_mistral': 3149, 'n_tokens_neox': 3069, 'n_words': 861}