<commit_before>var fs = require('fs');

var framesWorked = 0;
var frameCount = 1;
var frameContent = [];
var visited_pages = [];
var masterURL = "";
var masterPage = null;

function newIFrameLoad(page, load_iframes) {
  var frame_data = [];
  if (load_iframes) {
    frame_data = page.evaluate(function () {
      var framestmp = document.getElementsByTagName('IFRAME');
      var frames = [];
      for (var i=0;i<framestmp.length;i++) {
        if (framestmp[i].contentWindow && framestmp[i].contentWindow.document && framestmp[i].contentWindow.document.body) {
          frames.push(framestmp[i].contentWindow.document.body.innerHTML);
          framestmp[i].outerHTML = "<phantomjsframe>PHANTOM_JS_FRAME_"+i+"</phantomjsframe>";
        }
      }
      return frames;
    }) || []; //This stops the code from crashing incase there is an exception during page eval
  }
  var content = new String(page.content);
  for (var i=0;i<frame_data.length;i++) {
    content = content.replace("PHANTOM_JS_FRAME_"+i, frame_data[i]);
  }
  console.log("PHANTOMJS_DOMDATA_WRITE:"+content);
  console.log('PHANTOMJS_DOMDATA_END');
}

var loadpage = function(url, referer, success, failure, configure) {
  var redirectURL = null;

  var page = require('webpage').create();

  page.settings.localToRemoteUrlAccessEnabled = true;

  page.settings.resourceTimeout = 40000;

  page.onResourceTimeout = function(request) {
    console.log('Response (#' + request.id + '): ' + JSON.stringify(request));
  };

  page.settings.userAgent = "Mozilla/5.0 (X11; Linux i686) AppleWebKit/537.36 (KHTML, like Gecko) Ubuntu Chromium/34.0.1847.116 Chrome/34.0.1847.116 Safari/537.36";

  page.onConsoleMessage = function (msg) { console.log(msg); };

  page.onAlert = function(msg) { console.log(msg);};

  page.onLoadStarted = function () {
    console.log('Start loading...'+url);
  };

  page.onResourceReceived = function(resource) {
    if (url == resource.url && resource.redirectURL && resource.stage == "end") {
      redirectURL = resource.redirectURL;
      console.log('URL_FRAME_ERROR_CODE: '+resource.status+'URL_FRAME_ERROR_CODE_END');
    } else if (url == resource.url && resource.stage == "end" && resource.status != 200 && (referer!=undefined || !resource.redirectURL)) {
      console.log('URL_ERROR_CODE: '+resource.status+'URL_ERROR_CODE_END');
    }
  };

  if (referer!=undefined) {
    console.log("Set Referer: "+referer);
    page.customHeaders = {
      "REFERER": referer
    };
  }

  if (configure != undefined)
    configure(page);

  page.viewportSize = { width: 1920, height: 1080 };
  page.open(url, function (status) {
    console.log("Page Status: "+status);
    if (redirectURL) {
      loadpage(redirectURL, url, success, failure);
    } else if (status !== 'success') {
      failure(page, url);
    } else {
      success(page, url);
    }
  });
};

function main() {

  if (phantom.args.length < 2) {
      console.log('Usage: proxy.js <picture filename or none> <load iframe(true/false)> <URL> <url param count> <url params...>');
      phantom.exit();
  } else {
    file_name = phantom.args[0];
    var loadIframes = phantom.args[1].match(/true/i) ? true : false;
    address = phantom.args[2];

    var argCount = phantom.args[3];

    args = ""

    for (var i=0;i<argCount;i++) {
      args += phantom.args[i+4];
      if (i<argCount-1) args += "&"
    }
    if (args.length > 0)
      address += '?'+args;

    console.log("Open page: "+address+", "+args+" END");

    console.log('start openning page');

    masterURL = address;

    loadpage(address, undefined, function(page) {
      masterPage = page;
      console.log('DONE_LOADING_URL');<commit_msg>use system.args instead of deprecated phantom.args<commit_after>var fs = require('fs');
var system = require('system');

var framesWorked = 0;
var frameCount = 1;
var frameContent = [];
var visited_pages = [];
var masterURL = "";
var masterPage = null;

function newIFrameLoad(page, load_iframes) {
  var frame_data = [];
  if (load_iframes) {
    frame_data = page.evaluate(function () {
      var framestmp = document.getElementsByTagName('IFRAME');
      var frames = [];
      for (var i=0;i<framestmp.length;i++) {
        if (framestmp[i].contentWindow && framestmp[i].contentWindow.document && framestmp[i].contentWindow.document.body) {
          frames.push(framestmp[i].contentWindow.document.body.innerHTML);
          framestmp[i].outerHTML = "<phantomjsframe>PHANTOM_JS_FRAME_"+i+"</phantomjsframe>";
        }
      }
      return frames;
    }) || []; //This stops the code from crashing incase there is an exception during page eval
  }
  var content = new String(page.content);
  for (var i=0;i<frame_data.length;i++) {
    content = content.replace("PHANTOM_JS_FRAME_"+i, frame_data[i]);
  }
  console.log("PHANTOMJS_DOMDATA_WRITE:"+content);
  console.log('PHANTOMJS_DOMDATA_END');
}

var loadpage = function(url, referer, success, failure, configure) {
  var redirectURL = null;

  var page = require('webpage').create();

  page.settings.localToRemoteUrlAccessEnabled = true;

  page.settings.resourceTimeout = 40000;

  page.onResourceTimeout = function(request) {
    console.log('Response (#' + request.id + '): ' + JSON.stringify(request));
  };

  page.settings.userAgent = "Mozilla/5.0 (X11; Linux i686) AppleWebKit/537.36 (KHTML, like Gecko) Ubuntu Chromium/34.0.1847.116 Chrome/34.0.1847.116 Safari/537.36";

  page.onConsoleMessage = function (msg) { console.log(msg); };

  page.onAlert = function(msg) { console.log(msg);};

  page.onLoadStarted = function () {
    console.log('Start loading...'+url);
  };

  page.onResourceReceived = function(resource) {
    if (url == resource.url && resource.redirectURL && resource.stage == "end") {
      redirectURL = resource.redirectURL;
      console.log('URL_FRAME_ERROR_CODE: '+resource.status+'URL_FRAME_ERROR_CODE_END');
    } else if (url == resource.url && resource.stage == "end" && resource.status != 200 && (referer!=undefined || !resource.redirectURL)) {
      console.log('URL_ERROR_CODE: '+resource.status+'URL_ERROR_CODE_END');
    }
  };

  if (referer!=undefined) {
    console.log("Set Referer: "+referer);
    page.customHeaders = {
      "REFERER": referer
    };
  }

  if (configure != undefined)
    configure(page);

  page.viewportSize = { width: 1920, height: 1080 };
  page.open(url, function (status) {
    console.log("Page Status: "+status);
    if (redirectURL) {
      loadpage(redirectURL, url, success, failure);
    } else if (status !== 'success') {
      failure(page, url);
    } else {
      success(page, url);
    }
  });
};

function main() {
  console.log(system.args)
  if (system.args.length < 3) {
      console.log('Usage: proxy.js <picture filename or none> <load iframe(true/false)> <URL> <url param count> <url params...>');
      phantom.exit();
  } else {
    file_name = system.args[1];
    var loadIframes = system.args[2].match(/true/i) ? true : false;
    address = system.args[3];

    var argCount = system.args[4];

    args = ""

    for (var i=0;i<argCount;i++) {
      args += system.args[i+5];
      if (i<argCount-1) args += "&"
    }
    if (args.length > 0)
      address += '?'+args;

    console.log("Open page: "+address+", "+args+" END");

    console.log('start openning page');

    masterURL = address;

    loadpage(address, undefined, function(page) {
      masterPage = page;
      console.log('DONE_LOADING_URL');
{'fraction_non_alphanumeric': 0.16073108453164897, 'fraction_numerical': 0.017739551135600055, 'mean_word_length': 3.565644171779141, 'pattern_counts': {'":': 2, '<': 27, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2868075', 'n_tokens_mistral': 2585, 'n_tokens_neox': 2441, 'n_words': 490}