<commit_before>"use strict";

(function ($) {
$.getContent = function (source, options) {
	var settings = {
		type: 'auto',
		beforeComplete: $.noop,
		complete: $.noop,
		encoding: 'UTF-8', // Text encoding
		num: -1, // feed num
		timeout: 25 * 1000
	},
	timer,
	canceled = false,
	processFeedByDomain = {
		'pixnet.net': function (str, strType) {
			return str.replace(/\(.+?\.\.\.\)/g, '');
		},
		'wretch.cc': function (str, strType) {
			return str.replace(/本篇文章引用自此/g, '');
		}
	},
	pass = function (str, strType) {
		return str;
	},
	beforeComplete = function (str) {
		if (canceled) return;
		settings.beforeComplete(str);
	},
	complete = function (str) {
		if (canceled) return;
		clearTimeout(timer);
		settings.complete(str);
	},
	getFeedText = function () {
		return $.getJSON(
			'https://ajax.googleapis.com/ajax/services/feed/load?v=1.0&callback=?&scoring=h&num=' + settings.num.toString(10) + '&q=' + encodeURIComponent(source),
			function (data, status) {
				if (!data.responseData) {
					complete('');
					return;
				}

				beforeComplete(data.responseData.feed.title);
				var text = [],
				process = processFeedByDomain[source.match(/(\w+.\w+)\//)[1].toLowerCase()] || pass;

				data.responseData.feed.entries.forEach(
					function (entry) {
						text.push(process(entry.title, 'title'));
						text.push(process(entry.content, 'content').replace(/<[^>]+?>|\(.+?\.\.\.\)|\&\w+\;|<script.+?\/script\>/ig, ''));
					}
				);
				text = text.join('\n');
				setTimeout(
					function () {
						complete(text);
					},
					0
				);
			}
		);
	},
	getHTMLText = function () {
		$.getJSON(
			'http://query.yahooapis.com/v1/public/yql?q=select%20*%20from%20html%20where%20url%3D%22' + encodeURIComponent(source) + '%22&format=json&diagnostics=true&callback=?',
			function (data, status) {
				if (!data.query.results) {
					complete('');
					return;
				}
				beforeComplete('');
				var text = [];
				
				parseYQLElementObject(text, data.query.results);
				text = text.join('\n');
				setTimeout(
					function () {
						complete(text);
					},
					0
				);
			}
		);
	},
	getFbText = function () {
		var text = [],
		statusQuery = FB.Data.query('select message from status where uid = {0}', source),
		noteQuery = FB.Data.query('select content,title from note where uid = {0}', source),
		linkQuery = FB.Data.query('select owner_comment from link where owner = {0}', source);

		FB.Data.waitOn(
			[statusQuery, noteQuery, linkQuery],
			function() {
				beforeComplete('');
				statusQuery.value.forEach(
					function (row) {
						text.push(row.message);
					}
				);
				noteQuery.value.forEach(
					function (row) {
						text.push(row.content);
						text.push(row.title);
					}
				);
				linkQuery.value.forEach(
					function (row) {
						text.push(row.owner_comment);
					}
				);
				text = text.join('\n');
				setTimeout(
					function () {
						complete(text);
					},
					0
				);
			}
		);
	},
	parseYQLElementObject = function (text, obj) {
		// TBD, properly exclude script
		for (var key in obj) if (obj.hasOwnProperty(key)) {
			if (key === 'script' || key === 'style') continue;
			else if (key === 'content') text.push(obj[key]);
			else if (typeof obj[key] === 'object' && key !== 'script' && key !== 'style') parseYQLElementObject(text, obj[key]);
		}
	},
	getFileText = function (stripHTML) {
		// we read and process file one at a time, no place to fire beforeComplete()
		beforeComplete();
		var text = [], i = source.length - 1;<commit_msg>Strip HTML in Facebook note<commit_after>"use strict";

(function ($) {
$.getContent = function (source, options) {
	var settings = {
		type: 'auto',
		beforeComplete: $.noop,
		complete: $.noop,
		encoding: 'UTF-8', // Text encoding
		num: -1, // feed num
		timeout: 25 * 1000
	},
	timer,
	canceled = false,
	processFeedByDomain = {
		'pixnet.net': function (str, strType) {
			return str.replace(/\(.+?\.\.\.\)/g, '');
		},
		'wretch.cc': function (str, strType) {
			return str.replace(/本篇文章引用自此/g, '');
		}
	},
	pass = function (str, strType) {
		return str;
	},
	beforeComplete = function (str) {
		if (canceled) return;
		settings.beforeComplete(str);
	},
	complete = function (str) {
		if (canceled) return;
		clearTimeout(timer);
		settings.complete(str);
	},
	getFeedText = function () {
		return $.getJSON(
			'https://ajax.googleapis.com/ajax/services/feed/load?v=1.0&callback=?&scoring=h&num=' + settings.num.toString(10) + '&q=' + encodeURIComponent(source),
			function (data, status) {
				if (!data.responseData) {
					complete('');
					return;
				}

				beforeComplete(data.responseData.feed.title);
				var text = [],
				process = processFeedByDomain[source.match(/(\w+.\w+)\//)[1].toLowerCase()] || pass;

				data.responseData.feed.entries.forEach(
					function (entry) {
						text.push(process(entry.title, 'title'));
						text.push(process(entry.content, 'content').replace(/<[^>]+?>|\(.+?\.\.\.\)|\&\w+\;|<script.+?\/script\>/ig, ''));
					}
				);
				text = text.join('\n');
				setTimeout(
					function () {
						complete(text);
					},
					0
				);
			}
		);
	},
	getHTMLText = function () {
		$.getJSON(
			'http://query.yahooapis.com/v1/public/yql?q=select%20*%20from%20html%20where%20url%3D%22' + encodeURIComponent(source) + '%22&format=json&diagnostics=true&callback=?',
			function (data, status) {
				if (!data.query.results) {
					complete('');
					return;
				}
				beforeComplete('');
				var text = [];
				
				parseYQLElementObject(text, data.query.results);
				text = text.join('\n');
				setTimeout(
					function () {
						complete(text);
					},
					0
				);
			}
		);
	},
	getFbText = function () {
		var text = [],
		statusQuery = FB.Data.query('select message from status where uid = {0}', source),
		noteQuery = FB.Data.query('select content,title from note where uid = {0}', source),
		linkQuery = FB.Data.query('select owner_comment from link where owner = {0}', source);

		FB.Data.waitOn(
			[statusQuery, noteQuery, linkQuery],
			function() {
				beforeComplete('');
				statusQuery.value.forEach(
					function (row) {
						text.push(row.message);
					}
				);
				noteQuery.value.forEach(
					function (row) {
						text.push(row.content.replace(/<[^>]+?>|<script.+?\/script\>/ig, ''));
						text.push(row.title);
					}
				);
				linkQuery.value.forEach(
					function (row) {
						text.push(row.owner_comment);
					}
				);
				text = text.join('\n');
				setTimeout(
					function () {
						complete(text);
					},
					0
				);
			}
		);
	},
	parseYQLElementObject = function (text, obj) {
		// TBD, properly exclude script
		for (var key in obj) if (obj.hasOwnProperty(key)) {
			if (key === 'script' || key === 'style') continue;
			else if (key === 'content') text.push(obj[key]);
			else if (typeof obj[key] === 'object' && key !== 'script' && key !== 'style') parseYQLElementObject(text, obj[key]);
		}
	},
	getFileText = function (stripHTML) {
		// we read and process file one at a time, no place to fire beforeComplete()
		beforeComplete();
		var text = [], i = source.length - 1;
{'fraction_non_alphanumeric': 0.19040225829216656, 'fraction_numerical': 0.010162314749470713, 'mean_word_length': 2.934480843975569, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 12, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5132667', 'n_tokens_mistral': 3047, 'n_tokens_neox': 2461, 'n_words': 477}