<commit_before>/* This Source Code Form is subject to the terms of the Mozilla Public
 * License, v. 2.0. If a copy of the MPL was not distributed with this
 * file, You can obtain one at https://mozilla.org/MPL/2.0/. */
"use strict";

var df = {

	domainLookup: function(data){
		// Check if local domain or local ip is requested
		let special = df.isSpecial(data.url);
		if (special !== false) {
			return df.setFlag(df.deepExtend({}, special, data));
		}

		// parse url to a domain
		let domain = df.parseUrl(data.url);

		// If something is wrong with the domain, display a question symbol
		if (domain === false || domain === "" || domain === "false") {
			return df.setFlag({ tab: data.tab, icon: "images/fugue/question-white.png", title: "No data found", popup: 'special.html' });
		}

		if (typeof data.ip === "undefined") {
			data.ip = null;
		}

		// check if resolved ip is local
		if (data.ip != null) {
			let special = df.isInternal(data.ip);
			if (special !== false) {
				return df.setFlag(df.deepExtend({}, special, data));
			}
		}

		// check if given domain is stored within cache
		if (cache.has(domain)) {
			return df.domainLookupResultData({ lookup: data, request: cache.get(domain) })
		}

		if (requestQueue.has(domain)) {
			return setTimeout(function(){
				df.domainLookup(data);
			}, 250);
		}
		requestQueue.set(domain, true);

		// requested url is not special and not in cache, request data from server
		let request = new XMLHttpRequest();
		request.open('GET', api_protocol + '://' + api_domain + api_path + '/lookup/' + domain, true);

		// Timeout after 35 seconds
		request.timeout = 1000*35;
		request.ontimeout = function() {
			Sentry.withScope(function(scope) {
				scope.setExtra("server", api_domain);
				Sentry.captureMessage("timeout contacting server");
			});
			requestQueue.remove(domain);
			api_domain = api_domain_fallback;
		};

		request.onload = function() {
			if (this.status == 200) {
				let parsedData;
				try {
					parsedData = JSON.parse(this.response);
				}
				catch (e){
					let status = this.status;
					let response = this.response;
					Sentry.withScope(function(scope) {
						scope.setExtra("domain", domain);
						scope.setExtra("status", status);
						scope.setExtra("response", response);
						Sentry.captureException(e);
					});
					api_domain = api_domain_fallback;
				}
				if (parsedData === false) {
					// Something went wrong contacting the server
					let status = this.status;
					let response = this.response;
					Sentry.withScope(function(scope) {
						scope.setExtra("domain", domain);
						scope.setExtra("status", status);
						scope.setExtra("response", response);
						scope.setExtra("data", parsedData);
						Sentry.captureMessage("error parsing json response");
					});
					api_domain = api_domain_fallback;
					// TODO: error symbol
					return;
				}
				// Set cache and pass to result
				cache.set(domain, parsedData);
				df.domainLookupResultData({lookup: data, request: parsedData});
				requestQueue.remove(domain);
			} else {
				// Something went wrong contacting the server
				let status = this.status;
				let response = this.response;
				Sentry.withScope(function(scope) {
					scope.setExtra("domain", domain);
					scope.setExtra("status", status);
					scope.setExtra("response", response);
					scope.setExtra("server", api_domain);
					Sentry.captureMessage("error requesting data from server");
				});
				requestQueue.remove(domain);
				api_domain = api_domain_fallback;
				// TODO: error symbol
				return;
			}
		};

		request.onerror = function() {
			requestQueue.remove(domain);
			// There was a connection error of some sort
			// this is nothing we should report because we do not have error data
			// or can do anything about it. mostly connection issues from the user
			// TODO: error symbol
			Sentry.withScope(function(scope) {
				scope.setExtra("server", api_domain);
				Sentry.captureMessage("error contacting server");
			});
			api_domain = api_domain_fallback;
		};

		request.send();<commit_msg>Log requested URL after redirect retry lookup on fallback<commit_after>/* This Source Code Form is subject to the terms of the Mozilla Public
 * License, v. 2.0. If a copy of the MPL was not distributed with this
 * file, You can obtain one at https://mozilla.org/MPL/2.0/. */
"use strict";

var df = {

	domainLookup: function(data){
		// Check if local domain or local ip is requested
		let special = df.isSpecial(data.url);
		if (special !== false) {
			return df.setFlag(df.deepExtend({}, special, data));
		}

		// parse url to a domain
		let domain = df.parseUrl(data.url);

		// If something is wrong with the domain, display a question symbol
		if (domain === false || domain === "" || domain === "false") {
			return df.setFlag({ tab: data.tab, icon: "images/fugue/question-white.png", title: "No data found", popup: 'special.html' });
		}

		if (typeof data.ip === "undefined") {
			data.ip = null;
		}

		// check if resolved ip is local
		if (data.ip != null) {
			let special = df.isInternal(data.ip);
			if (special !== false) {
				return df.setFlag(df.deepExtend({}, special, data));
			}
		}

		// check if given domain is stored within cache
		if (cache.has(domain)) {
			return df.domainLookupResultData({ lookup: data, request: cache.get(domain) })
		}

		// wait until lookup has finished and execute again
		// this prevents multiple lookups of the same domain when multiple tabs open (e.g. session restore)
		if (requestQueue.has(domain)) {
			return setTimeout(function(){
				df.domainLookup(data);
			}, 250);
		}
		requestQueue.set(domain, true);

		// requested url is not special and not in cache, request data from server
		let request = new XMLHttpRequest();
		request.open('GET', api_protocol + '://' + api_domain + api_path + '/lookup/' + domain, true);

		// Timeout after 35 seconds
		request.timeout = 1000*35;
		request.ontimeout = function() {
			Sentry.withScope(function(scope) {
				scope.setExtra("server", api_domain);
				Sentry.captureMessage("timeout contacting server");
			});
			requestQueue.remove(domain);
			api_domain = api_domain_fallback;
		};

		request.onload = function() {
			if (this.status == 200) {
				let parsedData;
				try {
					parsedData = JSON.parse(this.response);
				}
				catch (e){
					let status = this.status;
					let response = this.response;
					Sentry.withScope(function(scope) {
						scope.setExtra("domain", domain);
						scope.setExtra("status", status);
						scope.setExtra("responseURL", request.responseURL);
						scope.setExtra("response", response);
						Sentry.captureException(e);
					});
					// TODO: error symbol
					if (api_domain != api_domain_fallback) {
						api_domain = api_domain_fallback;
						requestQueue.remove(domain);
						df.domainLookup(data);
					}
					return;
				}
				if (parsedData === false || typeof parsedData === "undefined") {
					// Something went wrong contacting the server
					let status = this.status;
					let response = this.response;
					Sentry.withScope(function(scope) {
						scope.setExtra("domain", domain);
						scope.setExtra("status", status);
						scope.setExtra("responseURL", request.responseURL);
						scope.setExtra("response", response);
						Sentry.captureMessage("error parsing json response");
					});
					api_domain = api_domain_fallback;
					// TODO: error symbol
					return;
				}
				// Set cache and pass to result
				cache.set(domain, parsedData);
				df.domainLookupResultData({lookup: data, request: parsedData});
				requestQueue.remove(domain);
			} else {
				// Something went wrong contacting the server
				let status = this.status;
				let response = this.response;
				Sentry.withScope(function(scope) {
					scope.setExtra("domain", domain);
					scope.setExtra("status", status);
					scope.setExtra("response", response);
					scope.setExtra("responseURL", request.responseURL);
					scope.setExtra("server", api_domain);
					Sentry.captureMessage("error requesting data from server");
				});
				requestQueue.remove(domain);
				api_domain = api_domain_fallback;
				// TODO: error symbol
				return;
			}
		};

		request.onerror = function() {
			requestQueue.remove(domain);
			// There was a connection error of some sort
			// this is nothing we should report because we do not have error data
			// or can do anything about it. mostly connection issues from the user
			// TODO: error symbol
			Sentry.withScope(function(scope) {
				scope.setExtra("responseURL", request.responseURL);
				scope.setExtra("server", api_domain);
				Sentry.captureMessage("error contacting server");
			});
			api_domain = api_domain_fallback;
		};

		request.send();
{'fraction_non_alphanumeric': 0.12950631458094145, 'fraction_numerical': 0.0041331802525832375, 'mean_word_length': 3.2143202709240444, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6233543', 'n_tokens_mistral': 3212, 'n_tokens_neox': 2545, 'n_words': 736}