<commit_before>import storage from "./oembed-storage";
import regexes from "./regexes";
import providers from "./providers";

function getContent(regex) {
	return regex[0].match(regexes.content)[0].match(/['|"].*/)[0].slice(1);
}

function parseHTML(body) {
	const bodyString = body.replace(/(\r\n|\n|\r)/g, "");
	const res = bodyString.match(regexes.link);

	if (res !== null) {
		return res[0].match(regexes.matchHTTP)[0].replace(/&amp;/g, "&");
	}

	const oembed = {
		type: "rich"
	};

	const props = [ "title", "description" ];

	for (let i = 0; i < props.length; i++) {
		const match = bodyString.match(regexes.propertyRegex(props[i]));

		if (match) {
			oembed[props[i]] = getContent(match);
		}
	}

	const propsWithType = [ "width", "height" ];

	for (let i = 0; i < propsWithType.length; i++) {
		const types = [ "video", "image" ];

		for (let j = 0; j < types.length; j++) {
			const match = bodyString.match(regexes.propertyRegex(propsWithType[i], types[j]));

			if (match) {
				oembed[propsWithType[i]] = parseInt(getContent(match), 10);
			}
		}
	}

	const imageUrl = bodyString.match(regexes.image);

	if (imageUrl) {
		oembed.thumbnail_url = getContent(imageUrl);
	}

	if (!oembed.title) {
		const title = bodyString.match(regexes.title);

		if (title) {
			oembed.title = title[0].match(/[>][^<]*/)[0].slice(1);
		}
	}

	if (!oembed.description) {
		const description = bodyString.match(regexes.description);

		if (description) {
			oembed.description = description[0].match(regexes.content)[0].match(/['|"][^'|^"]*/)[0].slice(1);
		}
	}

	if (Object.keys(oembed).length) {
		return oembed;
	} else {
		throw new Error("No oEmbed data found");
	}
}

async function embed(url) {
	const res = await fetch(url);
	const body = await res.text();
	const parsed = parseHTML(body);

	let data;

	if (typeof parsed === "string") {
		data = await (await fetch(parsed)).json();
	} else {<commit_msg>Decode html characters for oembed<commit_after>import storage from "./oembed-storage";
import regexes from "./regexes";
import providers from "./providers";

function getContent(regex) {
	return regex[0].match(regexes.content)[0].match(/['|"].*/)[0].slice(1);
}

function decodeText(text) {
	return text
	.replace(/&lt;/g, "<")
	.replace(/&gt;/g, ">")
	.replace(/&amp;/g, "&")
	.replace(/&quot;/g, '"')
	.replace(/&nbsp;/g, " ")
	.replace(/&#(x?)(\d+);/g, (m, p1, p2) => String.fromCharCode(((p1 === "x") ? parseInt(p2, 16) : p2)));
}

function parseHTML(body) {
	const bodyString = body.replace(/(\r\n|\n|\r)/g, "");
	const res = bodyString.match(regexes.link);

	if (res !== null) {
		return res[0].match(regexes.matchHTTP)[0].replace(/&amp;/g, "&");
	}

	const oembed = {
		type: "rich"
	};

	const props = [ "title", "description" ];

	for (let i = 0; i < props.length; i++) {
		const match = bodyString.match(regexes.propertyRegex(props[i]));

		if (match && match.length) {
			oembed[props[i]] = decodeText(getContent(match));
		}
	}

	const propsWithType = [ "width", "height" ];

	for (let i = 0; i < propsWithType.length; i++) {
		const types = [ "video", "image" ];

		for (let j = 0; j < types.length; j++) {
			const match = bodyString.match(regexes.propertyRegex(propsWithType[i], types[j]));

			if (match && match.length) {
				oembed[propsWithType[i]] = parseInt(getContent(match), 10);
			}
		}
	}

	const imageUrl = bodyString.match(regexes.image);

	if (imageUrl) {
		oembed.thumbnail_url = getContent(imageUrl);
	}

	if (!oembed.title) {
		const match = bodyString.match(regexes.title);

		if (match && match.length) {
			const title = title[0].match(/[>][^<]*/);

			if (title && title.length) {
				oembed.title = decodeText(title[0].slice(1));
			}
		}
	}

	if (!oembed.description) {
		const match = bodyString.match(regexes.description);

		if (match && match.length) {
			const description = description[0].match(regexes.content)[0].match(/['|"][^'|^"]*/);

			if (description && description.length) {
				oembed.description = decodeText(description[0].slice(1));
			}
		}
	}

	if (Object.keys(oembed).length) {
		return oembed;
	} else {
		throw new Error("No oEmbed data found");
	}
}

async function embed(url) {
	const res = await fetch(url);
	const body = await res.text();
	const parsed = parseHTML(body);

	let data;

	if (typeof parsed === "string") {
		data = await (await fetch(parsed)).json();
	} else {
{'fraction_non_alphanumeric': 0.20418487008507702, 'fraction_numerical': 0.009887330420786388, 'mean_word_length': 3.92638731596829, 'pattern_counts': {'":': 0, '<': 12, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2708041', 'n_tokens_mistral': 1697, 'n_tokens_neox': 1569, 'n_words': 324}