<commit_before>// Connection object
//
// Convert an HTTP request (channel) to a loggable, visualizable connection object, if possible

const { Cc, Ci, Cr } = require('chrome');
const Request = require('sdk/request').Request;
const timers = require('sdk/timers');
const { on, once, off, emit } = require('sdk/event/core');
const ss = require('sdk/simple-storage');
const { data } = require("sdk/self");
const addonUrl = data.url("index.html");

var eTLDSvc = Cc["@mozilla.org/network/effective-tld-service;1"].
                getService(Ci.nsIEffectiveTLDService);

const { getTabForChannel } = require('./tab/utils');

var captureAndStoreConnections = true;

exports.Connection = Connection;
exports.addConnection = addConnection;

// FIXME: Move persistence into a component
/* BEGIN FLAG PERSISTENCE */
var restored = false;
var storage = ss.storage;
function restore(){
    // only called when add-on is initialized, not when ui page is refreshed
    Connection.emit('restored');
    if ( storage.tempConnections && storage.tempConnections.length > 0 ){
        Connection.emit("tempConnections", storage.tempConnections);
    }
    restored = true;

    console.error('restored ' + Connection.restoredLogs + ' logs, ' + Connection.restoredObjects + ' objects');
}

function reset(){
    if (storage.tempConnections){
        storage.tempConnections.length = 0;
    }
}

console.log('Current capacity is %s% of quota', Math.round(ss.quotaUsage * 100));
/* END FLAG PERSISTENCE */

function addConnection(connection){
    Connection.emit("connection", connection.toLog()); // pass connection as array
    if ( !storage.tempConnections ) storage.tempConnections = [];
    if ( captureAndStoreConnections ){
        storage.tempConnections.push(connection.toLog());
        console.error('addConnection() ' + connection.toLog());
    }
}

function getDomain(host) {
  try {
    return eTLDSvc.getBaseDomainFromHost(host);
  } catch (e if e.result === Cr.NS_ERROR_INSUFFICIENT_DOMAIN_LEVELS) {
    return false;
  } catch (e if e.result === Cr.NS_ERROR_HOST_IS_IP_ADDRESS) {
    console.error('getDomain(): ip address: ' + host);
    return host;
  } catch (e){
    console.error('getDomain(): unexpected error: ' + host + ': ' + e);
    throw e;
  }
}

Connection.getDomain = getDomain; // make it part of what we export

function getSubdomain(host){
    var domain = getDomain(host);
    return host.slice(0, host.length - domain.length);
}

function Connection() {};

Connection.restoredLogs = 0;
Connection.restoredObjects = 0;

Connection.fromSubject = function(subject){
    var conn = new Connection();
    conn.restoreFromSubject(subject);
    return conn;
}

Connection.restore = function(obj){
    var conn = new Connection();
    if (Array.isArray(obj)){
        return obj;
    }else{
        conn.restoreFromObject(obj);
        return;
    }
};

Connection.prototype.restoreFromObject = function(obj){
    var self = this, host;
    Object.keys(obj).forEach(function(key){
        self[key] = obj[key];
    });
    // defaults for 1.1
    if (self.sourceSub === undefined){
        host = self.source;
        self.source = getDomain(host);
        self.sourceSub = getSubdomain(host);
    }
    if (self.targetSub === undefined){
        host = self.target;
        self.target = getDomain(host);
        self.targetSub = getSubdomain(host);
    }
    if (self.method === undefined){
        self.method = 'GET';
    }
    if (self.status === undefined){
        self.status = 200;
    }
    if (self.cacheable === undefined){
        self.cacheable = true;
    }
    self.valid = true;
    self.timestamp = new Date(obj.timestamp);
    Connection.restoredObjects += 1;
    return self
};

function getAjaxRequestHeader(channel){
    var header = null;
    try{
        header = channel.getRequestHeader('X-Requested-With').toLowerCase() === 'xmlhttprequest';
    }catch(e){
        if (e.name === 'NS_ERROR_NOT_AVAILABLE'){
            /* header not found, do nothing */
        }else{
            console.error('what is this? ' + Object.keys(e).join(','));
            throw e;
        }
    }
    return header;
}

function hasCookie(channel){
    try {
        return !! channel.getRequestHeader('Cookie');
    } catch (e) {
        return false;
    }
}

function getProtocol(uri){
    return uri.scheme;
}


Connection.prototype.restoreFromSubject = function(subject){
    // Check to see if this is in fact a third-party connection, if not, return
    var channel = subject.QueryInterface(Ci.nsIHttpChannel);
    var source = channel.referrer;
    var target = channel.URI;
    var targetDomain = getDomain(target.host);
    var tab = getTabForChannel(channel);
    var isAjax = getAjaxRequestHeader(channel);
    var valid = true;
    var browserUri = tab ? tab.linkedBrowser.currentURI : null;
    var browserSpec = browserUri && browserUri.spec;
    var browserDomain = null;
    try{
        var browserDomain = browserUri && getDomain(browserUri.host);
    }catch(e){
        // chances are the URL is about:blank, which has no host and throws an exception<commit_msg>fixed #378 with a try/catch<commit_after>// Connection object
//
// Convert an HTTP request (channel) to a loggable, visualizable connection object, if possible

const { Cc, Ci, Cr } = require('chrome');
const Request = require('sdk/request').Request;
const timers = require('sdk/timers');
const { on, once, off, emit } = require('sdk/event/core');
const ss = require('sdk/simple-storage');
const { data } = require("sdk/self");
const addonUrl = data.url("index.html");

var eTLDSvc = Cc["@mozilla.org/network/effective-tld-service;1"].
                getService(Ci.nsIEffectiveTLDService);

const { getTabForChannel } = require('./tab/utils');

var captureAndStoreConnections = true;

exports.Connection = Connection;
exports.addConnection = addConnection;

// FIXME: Move persistence into a component
/* BEGIN FLAG PERSISTENCE */
var restored = false;
var storage = ss.storage;
function restore(){
    // only called when add-on is initialized, not when ui page is refreshed
    Connection.emit('restored');
    if ( storage.tempConnections && storage.tempConnections.length > 0 ){
        Connection.emit("tempConnections", storage.tempConnections);
    }
    restored = true;

    console.error('restored ' + Connection.restoredLogs + ' logs, ' + Connection.restoredObjects + ' objects');
}

function reset(){
    if (storage.tempConnections){
        storage.tempConnections.length = 0;
    }
}

console.log('Current capacity is %s% of quota', Math.round(ss.quotaUsage * 100));
/* END FLAG PERSISTENCE */

function addConnection(connection){
    Connection.emit("connection", connection.toLog()); // pass connection as array
    if ( !storage.tempConnections ) storage.tempConnections = [];
    if ( captureAndStoreConnections ){
        storage.tempConnections.push(connection.toLog());
        console.error('addConnection() ' + connection.toLog());
    }
}

function getDomain(host) {
  try {
    return eTLDSvc.getBaseDomainFromHost(host);
  } catch (e if e.result === Cr.NS_ERROR_INSUFFICIENT_DOMAIN_LEVELS) {
    return false;
  } catch (e if e.result === Cr.NS_ERROR_HOST_IS_IP_ADDRESS) {
    console.error('getDomain(): ip address: ' + host);
    return host;
  } catch (e){
    console.error('getDomain(): unexpected error: ' + host + ': ' + e);
    throw e;
  }
}

Connection.getDomain = getDomain; // make it part of what we export

function getSubdomain(host){
    var domain = getDomain(host);
    return host.slice(0, host.length - domain.length);
}

function Connection() {};

Connection.restoredLogs = 0;
Connection.restoredObjects = 0;

Connection.fromSubject = function(subject){
    var conn = new Connection();
    conn.restoreFromSubject(subject);
    return conn;
}

Connection.restore = function(obj){
    var conn = new Connection();
    if (Array.isArray(obj)){
        return obj;
    }else{
        conn.restoreFromObject(obj);
        return;
    }
};

Connection.prototype.restoreFromObject = function(obj){
    var self = this, host;
    Object.keys(obj).forEach(function(key){
        self[key] = obj[key];
    });
    // defaults for 1.1
    if (self.sourceSub === undefined){
        host = self.source;
        self.source = getDomain(host);
        self.sourceSub = getSubdomain(host);
    }
    if (self.targetSub === undefined){
        host = self.target;
        self.target = getDomain(host);
        self.targetSub = getSubdomain(host);
    }
    if (self.method === undefined){
        self.method = 'GET';
    }
    if (self.status === undefined){
        self.status = 200;
    }
    if (self.cacheable === undefined){
        self.cacheable = true;
    }
    self.valid = true;
    self.timestamp = new Date(obj.timestamp);
    Connection.restoredObjects += 1;
    return self
};

function getAjaxRequestHeader(channel){
    var header = null;
    try{
        header = channel.getRequestHeader('X-Requested-With').toLowerCase() === 'xmlhttprequest';
    }catch(e){
        if (e.name === 'NS_ERROR_NOT_AVAILABLE'){
            /* header not found, do nothing */
        }else{
            console.error('what is this? ' + Object.keys(e).join(','));
            throw e;
        }
    }
    return header;
}

function hasCookie(channel){
    try {
        return !! channel.getRequestHeader('Cookie');
    } catch (e) {
        return false;
    }
}

function getProtocol(uri){
    return uri.scheme;
}


Connection.prototype.restoreFromSubject = function(subject){
    // Check to see if this is in fact a third-party connection, if not, return
    var channel = subject.QueryInterface(Ci.nsIHttpChannel);
    var source = channel.referrer;
    var target = channel.URI;
    var targetDomain = getDomain(target.host);
    try{
        var tab = getTabForChannel(channel);
    }catch(e){
        console.log('EXCEPTION CAUGHT: No tab for connection');
        tab = null;
    }
    var isAjax = getAjaxRequestHeader(channel);
    var valid = true;
    var browserUri = tab ? tab.linkedBrowser.currentURI : null;
    var browserSpec = browserUri && browserUri.spec;
    var browserDomain = null;
    try{
        var browserDomain = browserUri && getDomain(browserUri.host);
    }catch(e){
        // chances are the URL is about:blank, which has no host and throws an exception
{'fraction_non_alphanumeric': 0.13105385509492445, 'fraction_numerical': 0.0031964354901201086, 'mean_word_length': 3.1415964701163257, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3552887', 'n_tokens_mistral': 3080, 'n_tokens_neox': 3042, 'n_words': 759}