<commit_before>// Copyright 2014 LinkedIn Corp. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

/**
 * This script defines a browser, which views a url and logs the all the network traffic
 * including the requests made by the urls. Any accesses to private network is blocked
 * automatically.
 */

var fs = require('fs'),
    webpage = require('webpage'),
    system = require('system');

var browser = {};

(function(browser) {

  /**
   * Merge objects. Elements in the `dst` object will be overridden
   * if the same field exists in the `src` object.
   *
   * @param {Object} dst The destination object.
   * @param {Object} src The source object.
   */
  Object.extend = function(dst, src) {
    for (var prop in src) {
      dst[prop] = src[prop];
    }
    return dst;
  };

  /**
   * Timeout to exit the browser after making no requests.
   */
  var IDLE_TIMEOUT = 5000;

  /**
   * Timeout to end the connection to a request.
   */
  var REQUEST_TIMEOUT = 3000;

  var exitTimeoutHandle = null;

  var options = {
    useCookie: true,
    url: null,
    logFile: null,
    javascriptEnabled: true,
    hostedLocally: true,
    cookieDir: null,
    debug: false,
    ipLookupUrl: null
  };

  var pageResources = null;

 /**
  * Call the callback function after the idle timeout.
  *
  * @param {Function} callback The function to be called after the idle timeout.
  */
  var resetExitTimeoutHandle = function(callback, timeout) {

    clearTimeout(exitTimeoutHandle);

    var timeoutStartDate = new Date();
    var _timeout = timeout != null ? timeout : IDLE_TIMEOUT;

    exitTimeoutHandle = setTimeout(function() {
      var remainingTime = _timeout - (new Date().getTime() - timeoutStartDate.getTime());
      if(remainingTime > 0 ) {
        resetExitTimeoutHandle(callback, remainingTime);
      } else if(callback) {
        setTimeout(callback, 0);
      } else {
        exit();
      }
    }, _timeout);
  };

  /**
   * Return true if the `ip` is an IP address in a private network.
   *
   * @param {String} ip The string that represents an IP address.
   * @return {boolean} true if the `ip` represents an IP address in a private network.
   */
  var isPrivateIp = function(ip) {
    return /^(127\.0\.0\.1|192\.168\.|172\.(1[6-9]|2[0-9]|3[0-1])\.|10\.)/.test(ip);
  };

  /**
   * Return true if the `url` is a location inside of a private network.
   *
   * @param {String} url The url.
   * @return {boolean} true if the `url` represents a location inside of a private network.
   */
  var isPrivateNetwork = function(url) {
    var ip = undefined;

    if(options.ipLookupUrl) {
      var xhr = new XMLHttpRequest();
      xhr.open('GET', options.ipLookupUrl + '?url=' + encodeURIComponent(url), false);
      xhr.onreadystatechange = function() {
        if (xhr.readyState == 4 && xhr.status == 200) {
          try {
            var json = JSON.parse(xhr.responseText);
            ip = json.ip;
          } catch(err) {}
        }
      };
      xhr.send(null);
    }
    return ip && isPrivateIp(ip);
  };

  /**
   * Initialize the page load.<commit_msg>Avoid redundant ip-lookup for data: and blob: schemas<commit_after>// Copyright 2014 LinkedIn Corp. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

/**
 * This script defines a browser, which views a url and logs the all the network traffic
 * including the requests made by the urls. Any accesses to private network is blocked
 * automatically.
 */

var fs = require('fs'),
    webpage = require('webpage'),
    system = require('system');

var browser = {};

(function(browser) {

  /**
   * Merge objects. Elements in the `dst` object will be overridden
   * if the same field exists in the `src` object.
   *
   * @param {Object} dst The destination object.
   * @param {Object} src The source object.
   */
  Object.extend = function(dst, src) {
    for (var prop in src) {
      dst[prop] = src[prop];
    }
    return dst;
  };

  /**
   * Timeout to exit the browser after making no requests.
   */
  var IDLE_TIMEOUT = 5000;

  /**
   * Timeout to end the connection to a request.
   */
  var REQUEST_TIMEOUT = 3000;

  var exitTimeoutHandle = null;

  var options = {
    useCookie: true,
    url: null,
    logFile: null,
    javascriptEnabled: true,
    hostedLocally: true,
    cookieDir: null,
    debug: false,
    ipLookupUrl: null
  };

  var pageResources = null;

 /**
  * Call the callback function after the idle timeout.
  *
  * @param {Function} callback The function to be called after the idle timeout.
  */
  var resetExitTimeoutHandle = function(callback, timeout) {

    clearTimeout(exitTimeoutHandle);

    var timeoutStartDate = new Date();
    var _timeout = timeout != null ? timeout : IDLE_TIMEOUT;

    exitTimeoutHandle = setTimeout(function() {
      var remainingTime = _timeout - (new Date().getTime() - timeoutStartDate.getTime());
      if(remainingTime > 0 ) {
        resetExitTimeoutHandle(callback, remainingTime);
      } else if(callback) {
        setTimeout(callback, 0);
      } else {
        exit();
      }
    }, _timeout);
  };

  /**
   * Return true if the `ip` is an IP address in a private network.
   *
   * @param {String} ip The string that represents an IP address.
   * @return {boolean} true if the `ip` represents an IP address in a private network.
   */
  var isPrivateIp = function(ip) {
    return /^(127\.0\.0\.1|192\.168\.|172\.(1[6-9]|2[0-9]|3[0-1])\.|10\.)/.test(ip);
  };

  /**
   * Return true if the `url` is a location inside of a private network.
   *
   * @param {String} url The url.
   * @return {boolean} true if the `url` represents a location inside of a private network.
   */
  var isPrivateNetwork = function(url) {
    var ip = undefined;

    if(url && !/^(data:|blob:)/.test(url) && options.ipLookupUrl) {
      var xhr = new XMLHttpRequest();
      xhr.open('GET', options.ipLookupUrl + '?url=' + encodeURIComponent(url), false);
      xhr.onreadystatechange = function() {
        if (xhr.readyState == 4 && xhr.status == 200) {
          try {
            var json = JSON.parse(xhr.responseText);
            ip = json.ip;
          } catch(err) {}
        }
      };
      xhr.send(null);
    }
    return ip && isPrivateIp(ip);
  };

  /**
   * Initialize the page load.
{'fraction_non_alphanumeric': 0.12137834036568214, 'fraction_numerical': 0.01350210970464135, 'mean_word_length': 2.65041067761807, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '77075', 'n_tokens_mistral': 2215, 'n_tokens_neox': 2113, 'n_words': 746}