<commit_before>// Set of the hostnames that are going to be blocked
var blocklistSet = new Set();
var whitelistSet =new Set();
const disconnectJSON = require('../data/disconnect.json');
const disconnectEntitylist = require('../data/disconnectEntitylist.json');
var {allHosts, canonicalizeHost} = require('../js/canonicalize.js');



parseJason();
//parseDisconnectEntity();

chrome.webRequest.onBeforeSendHeaders.addListener(function(details) {
    // do the blocking

    var areWeCancelling; // boolean checking blocking status
    var assetAdHost = canonicalizeHost(parseURI(details.url).hostname);

    if(isAd(details.url)) {
    	areWeCancelling = true;
    	console.log("bitch we be blockin " + assetAdHost);
    }


    return {cancel: areWeCancelling};
},{urls:["*://*/*"]}, ["blocking", "requestHeaders"]);

/*
* By Francesco
*
* Edited by <NAME>
* @returns true if it is an ad.
*/
function isAd(url) {
  // if url is something like ads.click.com then you would get all the hosts.. click.com
  var getHostAd = allHosts(parseURI(url).hostname);
  console.log("Before "+parseURI(url).hostname);
  console.log("After "+ getHostAd );

  if(whitelistSet.has(getHostAd)){
    //return false;
  }

for (var host in getHostAd){
 	if(blocklistSet.has(getHostAd[host])) {
		return true;
	}
}
return false;

}

function parseURI(url) {
    var match = url.match(/^((https|http)?\:)\/\/(([^:\/?#]*)(?:\:([0-9]+))?)(\/[^?#]*)(\?[^#]*|)(#.*|)$/);
    return match && {
        protocol: match[1],
        host: match[2],
        hostname: match[3],
        port: match[4],
        pathname: match[5],
        search: match[6],
        hash: match[7]
    }
}// end parse url


function parseJason(){

//delete disconnectJSON.categories['Content']
//delete disconnectJSON.categories['Legacy Disconnect']
//delete disconnectJSON.categories['Legacy Content']
	// parse our disconnect JSON into a set where we only include the hostname and subdomain urls
    for(var category in disconnectJSON.categories) {
      //  Advertising, Content ,Analytics, Social, Disconnect
	    for(var network in disconnectJSON.categories[category]) {
 	        for(var hostname in disconnectJSON.categories[category][network]) {
             // 2leep.com , 33Across , 4INFO ,4mads ...... and so on
	            blocklistSet.add(hostname); // add to the set
	            for(var subDomain in disconnectJSON.categories[category][network][hostname]) {
                // gets the subdomain as http://2leep.com/ , http://33across.com/ , http://www.4info.com/
	                for(var entitySubDomain in disconnectJSON.categories[category][network][hostname][subDomain]) {
                    // gets wierd random numbers
	                     blocklistSet.add(disconnectJSON.categories[category][network][hostname][subDomain][entitySubDomain]);
	                }
	            }
	        }
	      }
    }
<commit_msg>Fixed Jason. Poor Jason dude<commit_after>// Set of the hostnames that are going to be blocked
var blocklistSet = new Set();
var whitelistSet =new Set();
const disconnectJSON = require('../data/disconnect.json');
const disconnectEntitylist = require('../data/disconnectEntitylist.json');
var {allHosts, canonicalizeHost} = require('../js/canonicalize.js');



parseJSON();
//parseDisconnectEntity();

chrome.webRequest.onBeforeSendHeaders.addListener(function(details) {
    // do the blocking

    var areWeCancelling; // boolean checking blocking status
    var assetAdHost = canonicalizeHost(parseURI(details.url).hostname);

    if(isAd(details.url)) {
    	areWeCancelling = true;
    	console.log("bitch we be blockin " + assetAdHost);
    }


    return {cancel: areWeCancelling};
},{urls:["*://*/*"]}, ["blocking", "requestHeaders"]);

/*
* By Francesco
*
* Edited by <NAME>
* @returns true if it is an ad.
*/
function isAd(url) {
  // if url is something like ads.click.com then you would get all the hosts.. click.com
  var getHostAd = allHosts(parseURI(url).hostname);
  console.log("Before "+parseURI(url).hostname);
  console.log("After "+ getHostAd );

  if(whitelistSet.has(getHostAd)){
    //return false;
  }

for (var host in getHostAd){
 	if(blocklistSet.has(getHostAd[host])) {
		return true;
	}
}
return false;

}

function parseURI(url) {
    var match = url.match(/^((https|http)?\:)\/\/(([^:\/?#]*)(?:\:([0-9]+))?)(\/[^?#]*)(\?[^#]*|)(#.*|)$/);
    return match && {
        protocol: match[1],
        host: match[2],
        hostname: match[3],
        port: match[4],
        pathname: match[5],
        search: match[6],
        hash: match[7]
    }
}// end parse url


function parseJSON(){

//delete disconnectJSON.categories['Content']
//delete disconnectJSON.categories['Legacy Disconnect']
//delete disconnectJSON.categories['Legacy Content']
	// parse our disconnect JSON into a set where we only include the hostname and subdomain urls
    for(var category in disconnectJSON.categories) {
      //  Advertising, Content ,Analytics, Social, Disconnect
	    for(var network in disconnectJSON.categories[category]) {
 	        for(var hostname in disconnectJSON.categories[category][network]) {
             // 2leep.com , 33Across , 4INFO ,4mads ...... and so on
	            blocklistSet.add(hostname); // add to the set
	            for(var subDomain in disconnectJSON.categories[category][network][hostname]) {
                // gets the subdomain as http://2leep.com/ , http://33across.com/ , http://www.4info.com/
	                for(var entitySubDomain in disconnectJSON.categories[category][network][hostname][subDomain]) {
                    // gets wierd random numbers
	                     blocklistSet.add(disconnectJSON.categories[category][network][hostname][subDomain][entitySubDomain]);
	                }
	            }
	        }
	      }
    }

{'fraction_non_alphanumeric': 0.1512532411408816, 'fraction_numerical': 0.006222990492653414, 'mean_word_length': 3.063202247191011, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3043749', 'n_tokens_mistral': 1810, 'n_tokens_neox': 1719, 'n_words': 414}