<commit_before>// Runs a very simple crawl on an HTTP server

var chai = require("chai");
	chai.should();

describe("Crawler link discovery",function() {

	var Crawler = null,
		crawler = null,
		discover = null;

	beforeEach(function() {
		Crawler	= require("../");
		crawler = new Crawler();
		discover = crawler.discoverResources.bind(crawler);
	});

	it("should discover http/s prefixed URLs in the document",function() {

		var links =
			discover("	blah blah http://google.com/ \
						blah blah https://fish.com/resource blah \
						//example.com");

		links.should.be.an("array");
		links.length.should.equal(2);
		links[0].should.equal("http://google.com/");
		links[1].should.equal("https://fish.com/resource");
	});

	it("should discover URLS in quoted attributes in the document",function() {

		var links =
			discover("	<a href='google.com'> \
						<img src=\"http://example.com/resource with spaces.txt\"> \
						url('thingo.com/test.html')");

		links.should.be.an("array");
		links.length.should.equal(4);
		links[0].should.equal("google.com");
		links[1].should.equal("http://example.com/resource%20with%20spaces.txt");
		links[2].should.equal("thingo.com/test.html");
	});

	it("should discover URLS in unquoted attributes in the document",function() {

		var links =
			discover("	<a href=google.com> \
						<img src=http://example.com/resource with spaces.txt> \
						url(thingo.com/test.html)");

		links.should.be.an("array");
		links.length.should.equal(3);
		links[0].should.equal("google.com");
		links[1].should.equal("http://example.com/resource");
		links[2].should.equal("thingo.com/test.html");
	});

	it("should replace all '&amp;'s with ampersands",function() {

		var links =
			discover("<a href='http://example.com/resource?with&amp;query=params&amp;and=entities'>");

		links.should.be.an("array");
		links.length.should.equal(2);
		links[0].should.equal("http://example.com/resource?with&query=params&and=entities");
		links[1].should.equal("http://example.com/resource");
	});

	it("should ignore HTML comments with parseHTMLComments = false",function() {

		crawler.parseHTMLComments = false;

		var links =
			discover("	<!-- http://example.com/oneline_comment --> \
						<a href=google.com> \
						<!-- \
						http://example.com/resource \
						<a href=example.com> \
						-->");

		links.should.be.an("array");
		links.length.should.equal(1);
		links[0].should.equal("google.com");
	});

	it("should ignore script tags with parseScriptTags = false",function() {

		crawler.parseScriptTags = false;

		var links =
			discover("	<script>var a = \"<a href='http://example.com/oneline_script'></a>\";</script> \
						<a href=google.com> \
						<script type='text/javascript'> \
						http://example.com/resource \
						<a href=example.com> \
						</SCRIPT>");

		links.should.be.an("array");
		links.length.should.equal(1);
		links[0].should.equal("google.com");
	});

	it("should discover URLs legitimately ending with a quote or parenthesis",function() {

		var links =
			discover("<a href='example.com/resource?with(parentheses)'>\
						<a href='example.com/resource?with\"double quotes\"'>\
						<a href=\"example.com/resource?with'single quotes'\">");

		links.should.be.an("array");
		links.length.should.equal(3);
		links[0].should.equal("example.com/resource?with%28parentheses%29");
		links[1].should.equal("example.com/resource?with\"double quotes\"");
		links[2].should.equal("example.com/resource?with'single quotes'");
	});
});<commit_msg>Fix broken tests<commit_after>// Runs a very simple crawl on an HTTP server

var chai = require("chai");
	chai.should();

describe("Crawler link discovery",function() {

	var Crawler = null,
		crawler = null,
		discover = null;

	beforeEach(function() {
		Crawler	= require("../");
		crawler = new Crawler();
		discover = crawler.discoverResources.bind(crawler);
	});

	it("should discover http/s prefixed URLs in the document",function() {

		var links =
			discover("	blah blah http://google.com/ \
						blah blah https://fish.com/resource blah \
						//example.com");

		links.should.be.an("array");
		links.length.should.equal(2);
		links[0].should.equal("http://google.com/");
		links[1].should.equal("https://fish.com/resource");
	});

	it("should discover URLS in quoted attributes in the document",function() {

		var links =
			discover("	<a href='google.com'> \
						<img src=\"http://example.com/resource with spaces.txt\"> \
						url('thingo.com/test.html')");

		links.should.be.an("array");
		links.length.should.equal(4);
		links[0].should.equal("google.com");
		links[1].should.equal("http://example.com/resource%20with%20spaces.txt");
		links[2].should.equal("thingo.com/test.html");
	});

	it("should discover URLS in unquoted attributes in the document",function() {

		var links =
			discover("	<a href=google.com> \
						<img src=http://example.com/resource with spaces.txt> \
						url(thingo.com/test.html)");

		links.should.be.an("array");
		links.length.should.equal(3);
		links[0].should.equal("google.com");
		links[1].should.equal("http://example.com/resource");
		links[2].should.equal("thingo.com/test.html");
	});

	it("should replace all '&amp;'s with ampersands",function() {

		var links =
			discover("<a href='http://example.com/resource?with&amp;query=params&amp;and=entities'>");

		links.should.be.an("array");
		links.length.should.equal(2);
		links[0].should.equal("http://example.com/resource?with&query=params&and=entities");
		links[1].should.equal("http://example.com/resource");
	});

	it("should ignore HTML comments with parseHTMLComments = false",function() {

		crawler.parseHTMLComments = false;

		var links =
			discover("	<!-- http://example.com/oneline_comment --> \
						<a href=google.com> \
						<!-- \
						http://example.com/resource \
						<a href=example.com> \
						-->");

		links.should.be.an("array");
		links.length.should.equal(1);
		links[0].should.equal("google.com");
	});

	it("should ignore script tags with parseScriptTags = false",function() {

		crawler.parseScriptTags = false;

		var links =
			discover("	<script>var a = \"<a href='http://example.com/oneline_script'></a>\";</script> \
						<a href=google.com> \
						<script type='text/javascript'> \
						http://example.com/resource \
						<a href=example.com> \
						</SCRIPT>");

		links.should.be.an("array");
		links.length.should.equal(1);
		links[0].should.equal("google.com");
	});

	it("should discover URLs legitimately ending with a quote or parenthesis",function() {

		var links =
			discover("<a href='example.com/resource?with(parentheses)'>\
						<a href='example.com/resource?with\"double quotes\"'>\
						<a href=\"example.com/resource?with'single quotes'\">");

		links.should.be.an("array");
		links.length.should.equal(3);
		links[0].should.equal("example.com/resource?with%28parentheses%29");
		links[1].should.equal("example.com/resource?with%22double+quotes%22");
		links[2].should.equal("example.com/resource?with%27single+quotes%27");
	});
});
{'fraction_non_alphanumeric': 0.19473309608540926, 'fraction_numerical': 0.009679715302491104, 'mean_word_length': 3.809034907597536, 'pattern_counts': {'":': 0, '<': 43, '<?xml version=': 0, '>': 43, 'https://': 4, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3780711', 'n_tokens_mistral': 2713, 'n_tokens_neox': 2338, 'n_words': 384}