<commit_before>/*
 * Copyright (c) 2013, <NAME> <<EMAIL>>
 *  All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions are met:
 * 
 * - Redistributions of source code must retain the above copyright notice,
 *   this list of conditions and the following disclaimer.
 * 
 * - Redistributions in binary form must reproduce the above copyright notice,
 *   this list of conditions and the following disclaimer in the documentation
 *   and/or other materials provided with the distribution.
 * 
 * - Neither the name of iBot nor the names of its contributors may be used to
 *   endorse or promote products derived from this software without specific
 *   prior written permission.
 * 
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
 * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
 * POSSIBILITY OF SUCH DAMAGE.
 */

var http = require('http');

exports.mod = function(context)
{
	this.active = false;

	this._load = function(data)
	{
		if(data.active !== undefined && typeof data.active === 'boolean')
		{
			this.active = data.active;
		}
	}

	this._save = function()
	{
		return { active: this.active };
	}

	this.core$cmd = function(server, prefix, target, cmd, params)
	{
		if(cmd === 'scraper')
		{
			switch(params[0])
			{
				case 'on':
					if(server.master.test(prefix.mask))
					{
						this.active = true;
						server.send('PRIVMSG ' + target + ' :Scrape command is now active.');
					}
					break;
				case 'off':
					if(server.master.test(prefix.mask))
					{
						this.active = false;
						server.send('PRIVMSG ' + target + ' :Scrape command is now inactive.');
					}
					break;
			}
		}
		else if(cmd === 'scrape' &&  this.active)
		{
			var host = params[0];
			var path = params[1];
			var expr = params[2];

			server.send('PRIVMSG ' + target + ' :Scraping: ' + host + path);

			server.do('scraper$scrape', server, target, host, path, 'Data', new RegExp(expr), function(server, target, key, value)
			{
				if(value !== undefined) server.send('PRIVMSG ' + target + ' :' + key + ': ' + value);
				else server.send('PRIVMSG ' + target + ' :Scrape failed.');
			});
		}
	}

	this._scrape = function(server, target, host, path, key, expr, callback, processor)
	{
		if(processor === undefined)
		{
			processor = function(match) { if(match !== null) return match[1]; }
		}

		this.get(host, path, function(data)
		{
			if(data !== undefined)
			{
				var match = data.match(expr);
				callback(server, target, key, processor(match));
			}
			else callback(server, target, key, undefined);
		}.bind(this));
	}

	this.get = function(host, path, callback, depth)
	{
		if(depth === undefined) depth = 0;
		if(depth > 5) { callback(undefined); return; }

		var options = { host: host, path: path };

		var req = http.request(options, function(response)
		{
			response.on('error', function(err)
			{
				console.log(err);
				callback(undefined);
			});

			switch(response.statusCode)
			{
				case 301:
					// TODO: recursion protection (max 5 including initial request)
					this.get(response.headers.location, callback, ++depth);
					break;
				case 200:
					var data = '';
					response.on('data', function(chunk)
					{
						data += chunk;
						if(data.length > 10000000) req.abort();
					});
					response.on('end', function() { callback(data); });
					break;
				default:
					console.log(response.statusCode);
					break;
			}
		}.bind(this));

		req.on('error', function(err)
		{
			console.log(err);
			callback(undefined);
		});

		req.end();
	}
}<commit_msg>Refactored scraper module more to handle errors properly.<commit_after>/*
 * Copyright (c) 2013, <NAME> <<EMAIL>>
 *  All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions are met:
 * 
 * - Redistributions of source code must retain the above copyright notice,
 *   this list of conditions and the following disclaimer.
 * 
 * - Redistributions in binary form must reproduce the above copyright notice,
 *   this list of conditions and the following disclaimer in the documentation
 *   and/or other materials provided with the distribution.
 * 
 * - Neither the name of iBot nor the names of its contributors may be used to
 *   endorse or promote products derived from this software without specific
 *   prior written permission.
 * 
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
 * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
 * POSSIBILITY OF SUCH DAMAGE.
 */

var http = require('http');

exports.mod = function(context)
{
	this.active = false;

	this._load = function(data)
	{
		if(data.active !== undefined && typeof data.active === 'boolean')
		{
			this.active = data.active;
		}
	}

	this._save = function()
	{
		return { active: this.active };
	}

	this.core$cmd = function(server, prefix, target, cmd, params)
	{
		if(cmd === 'scraper')
		{
			switch(params[0])
			{
				case 'on':
					if(server.master.test(prefix.mask))
					{
						this.active = true;
						server.send('PRIVMSG ' + target + ' :Scrape command is now active.');
					}
					break;
				case 'off':
					if(server.master.test(prefix.mask))
					{
						this.active = false;
						server.send('PRIVMSG ' + target + ' :Scrape command is now inactive.');
					}
					break;
			}
		}
		else if(cmd === 'scrape' &&  this.active)
		{
			var host = params[0];
			var path = params[1];
			var expr = params[2];

			server.send('PRIVMSG ' + target + ' :Scraping: ' + host + path);

			server.do('scraper$scrape', server, target, host, path, 'Data', new RegExp(expr), function(server, target, key, value)
			{
				if(value !== undefined) server.send('PRIVMSG ' + target + ' :' + key + ': ' + value);
				else server.send('PRIVMSG ' + target + ' :Scrape failed.');
			});
		}
	}

	this._scrape = function(server, target, host, path, key, expr, callback, processor)
	{
		if(processor === undefined)
		{
			processor = function(match) { if(match !== null) return match[1]; }
		}

		this.get(host, path, function(data)
		{
			if(data !== undefined)
			{
				var match = data.match(expr);
				callback(server, target, key, processor(match));
			}
			else callback(server, target, key, undefined);
		}.bind(this));
	}

	this.get = function(host, path, callback, depth)
	{
		if(depth === undefined) depth = 0;
		if(depth > 5) { callback(undefined); return; }

		var options = { host: host, path: path };

		var req = http.request(options, function(response)
		{
			response.on('error', function(err)
			{
				console.log(err);
				callback();
			});

			switch(response.statusCode)
			{
				case 301:
					this.get(host, response.headers.location, callback, ++depth);
					break;
				case 200:
					var data = '';

					response.on('data', function(chunk)
					{
						data += chunk;
						if(data.length > 10000000) req.abort();
					});
					response.on('end', function() { callback(data); });
					break;
				default:
					console.log(response.statusCode);
					callback();
					break;
			}
		}.bind(this));

		req.on('error', function(err)
		{
			console.log(err);
			callback();
		});

		setTimeout(function() { req.abort(); }, 10000);
		req.end();
	}
}
{'fraction_non_alphanumeric': 0.12799252598388416, 'fraction_numerical': 0.006539764101366343, 'mean_word_length': 3.250124069478908, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 2, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2926202', 'n_tokens_mistral': 3131, 'n_tokens_neox': 2531, 'n_words': 840}