<commit_before>// caniuse.com functionality for oftn-bot
// by ImBcmDth - <EMAIL>
//
// Thanks to FireFly, sorella, gkatsev, Morthchek,
// and eboy for your testing and suggestions.
//
// All data provided by the good folks at <http://caniuse.com/>

var Utils = require("util");
var File = require("fs");
var Path = require('path');
var HTTPS = require('https');

var agentsToShow = {
	'ie': 'IE',
	'firefox': 'FF',
	'chrome': 'Chrome',
	'opera': 'Opera',
	'safari': 'Safari',
	'ios_saf': 'iOS',
	'android': 'Android'
};

var agentSupportToColors = {
	'n': '\x034',
	'y': '\x033',
	'p': '\x038',
	'a': '\x038',
	'u': '\x03E',
	'x': '\x03E'
};

// attempt reload the data every 24 hours
var reloadInterval = 24 * 60 * 60 * 1000;

var CanIUseServer = module.exports = function(filename) {
	this.dataSource = {
		hostname: 'raw.github.com',
		port: 443,
		path: '/Fyrd/caniuse/master/data.json',
		method: 'GET'
	};
	this.loaded = false;
	this.attemptsLeft = 5;

	if (filename) {
		this.filename = filename;
		this.readJSON();
	} else {
		this.fetchJSON();
	}
};

CanIUseServer.prototype.readJSON = function() {
	File.readFile(this.filename, function (err, data) {
		if (err) {
			throw new Error("Error reading file: " + this.filename);
		}
		try {
			this.parseJSON(data, this.filename);
			setTimeout(this.readJSON.bind(this), reloadInterval);
		} catch (e) {
			if (this.attemptsLeft--) {
				Utils.puts("Will attempt to read the file again... (" + (5 - this.attemptsLeft) + "/5)");
				setTimeout(this.readJSON.bind(this), 1000);
			} else {
				Utils.puts("ERROR: Out of read attempts...");
			}
		}
	}.bind(this));
};

CanIUseServer.prototype.fetchJSON = function() {
	var json = '';

	var req = HTTPS.request(this.dataSource, function(res) {
		res.on('data', function(data) {
			json += data;
		}.bind(this));

		res.on('end', function() {
			try {
				this.parseJSON(json, this.dataSource.hostname + this.dataSource.path);
				setTimeout(this.fetchJSON.bind(this), reloadInterval);
			} catch (e) {
				if (this.attemptsLeft--) {
					Utils.puts("Will attempt to fetch the file again in 10 seconds... (" + (5 - this.attemptsLeft) + "/5)");
					setTimeout(this.fetchJSON.bind(this), 10000);
				} else {
					Utils.puts("ERROR: Out of fetch attempts...");
				}
			}
		}.bind(this));
	}.bind(this));

	req.end();
};

CanIUseServer.prototype.parseJSON = function(json, source) {
	try {
		Utils.puts("Loaded JSON: " + source);
		var data = JSON.parse(json);
		this.loaded = true;
		this.attemptsLeft = 5;
		this.db = data;
		this.buildIndex();
	} catch (e) {
		Utils.puts("JSON Parse Error: " + e);
		throw e;
	}
};

CanIUseServer.prototype.buildIndex = function() {
	var db = this.db.data;

	this.keys = Object.keys(db)
	this.index = this.keys.map(concatTitleKeywords);

	function concatTitleKeywords(key) {
		return db[key].title.toLowerCase() + ',' + db[key].keywords.toLowerCase();
	}
};

CanIUseServer.prototype.search = function(key) {
	key = key.toLowerCase();

	if (!this.loaded) {
		return;
	}

	try {
		return this.find(key);
	} catch(e) {
		var i = this.index.length;

		while (i--) {
			if (this.index[i].search(key) > -1) {
				return this.find(this.keys[i]);
			}
		}

		throw e;
	}
};

CanIUseServer.prototype.find = function(key) {
	var db = this.db.data;

	if (typeof db[key] === "undefined") {
		throw new Error("Can I Use `"+key+"` was not found.");
	}

	var feature = db[key];
	var agentSupportInfo = this.getAgentSupportArray(feature);
	var agentSupportStrings = this.formatAgentSupportArray(agentSupportInfo.supported);<commit_msg>Changed String.search to String.indexOf for substring matching.<commit_after>// caniuse.com functionality for oftn-bot
// by ImBcmDth - <EMAIL>
//
// Thanks to FireFly, sorella, gkatsev, Morthchek,
// and eboy for your testing and suggestions.
//
// All data provided by the good folks at <http://caniuse.com/>

var Utils = require("util");
var File = require("fs");
var Path = require('path');
var HTTPS = require('https');

var agentsToShow = {
	'ie': 'IE',
	'firefox': 'FF',
	'chrome': 'Chrome',
	'opera': 'Opera',
	'safari': 'Safari',
	'ios_saf': 'iOS',
	'android': 'Android'
};

var agentSupportToColors = {
	'n': '\x034',
	'y': '\x033',
	'p': '\x038',
	'a': '\x038',
	'u': '\x03E',
	'x': '\x03E'
};

// attempt reload the data every 24 hours
var reloadInterval = 24 * 60 * 60 * 1000;

var CanIUseServer = module.exports = function(filename) {
	this.dataSource = {
		hostname: 'raw.github.com',
		port: 443,
		path: '/Fyrd/caniuse/master/data.json',
		method: 'GET'
	};
	this.loaded = false;
	this.attemptsLeft = 5;

	if (filename) {
		this.filename = filename;
		this.readJSON();
	} else {
		this.fetchJSON();
	}
};

CanIUseServer.prototype.readJSON = function() {
	File.readFile(this.filename, function (err, data) {
		if (err) {
			throw new Error("Error reading file: " + this.filename);
		}
		try {
			this.parseJSON(data, this.filename);
			setTimeout(this.readJSON.bind(this), reloadInterval);
		} catch (e) {
			if (this.attemptsLeft--) {
				Utils.puts("Will attempt to read the file again... (" + (5 - this.attemptsLeft) + "/5)");
				setTimeout(this.readJSON.bind(this), 1000);
			} else {
				Utils.puts("ERROR: Out of read attempts...");
			}
		}
	}.bind(this));
};

CanIUseServer.prototype.fetchJSON = function() {
	var json = '';

	var req = HTTPS.request(this.dataSource, function(res) {
		res.on('data', function(data) {
			json += data;
		}.bind(this));

		res.on('end', function() {
			try {
				this.parseJSON(json, this.dataSource.hostname + this.dataSource.path);
				setTimeout(this.fetchJSON.bind(this), reloadInterval);
			} catch (e) {
				if (this.attemptsLeft--) {
					Utils.puts("Will attempt to fetch the file again in 10 seconds... (" + (5 - this.attemptsLeft) + "/5)");
					setTimeout(this.fetchJSON.bind(this), 10000);
				} else {
					Utils.puts("ERROR: Out of fetch attempts...");
				}
			}
		}.bind(this));
	}.bind(this));

	req.end();
};

CanIUseServer.prototype.parseJSON = function(json, source) {
	try {
		Utils.puts("Loaded JSON: " + source);
		var data = JSON.parse(json);
		this.loaded = true;
		this.attemptsLeft = 5;
		this.db = data;
		this.buildIndex();
	} catch (e) {
		Utils.puts("JSON Parse Error: " + e);
		throw e;
	}
};

CanIUseServer.prototype.buildIndex = function() {
	var db = this.db.data;

	this.keys = Object.keys(db)
	this.index = this.keys.map(concatTitleKeywords);

	function concatTitleKeywords(key) {
		return db[key].title.toLowerCase() + ',' + db[key].keywords.toLowerCase();
	}
};

CanIUseServer.prototype.search = function(key) {
	key = key.toLowerCase();

	if (!this.loaded) {
		return;
	}

	try {
		return this.find(key);
	} catch(e) {
		var i = this.index.length;

		while (i--) {
			if (this.index[i].indexOf(key) > -1) {
				return this.find(this.keys[i]);
			}
		}

		throw e;
	}
};

CanIUseServer.prototype.find = function(key) {
	var db = this.db.data;

	if (typeof db[key] === "undefined") {
		throw new Error("Can I Use `"+key+"` was not found.");
	}

	var feature = db[key];
	var agentSupportInfo = this.getAgentSupportArray(feature);
	var agentSupportStrings = this.formatAgentSupportArray(agentSupportInfo.supported);
{'fraction_non_alphanumeric': 0.181297976273552, 'fraction_numerical': 0.013677599441730635, 'mean_word_length': 3.5354430379746837, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6159888', 'n_tokens_mistral': 2910, 'n_tokens_neox': 2617, 'n_words': 583}