<commit_before>/* Collect info hashes to queue for indexing
 */
 
var needle = require("needle");
var gunzip = require("gunzip-maybe");
var url = require("url");

var importers = { 
	dump: require("../importers/dump"),
	xmlRss: require("../importers/xml-rss"),
	generic: require("../importers/generic")
};

var db = require("../lib/db");

function collect(source, callback, onHash)
{
	var status = { found: 0, start: Date.now() }, stream, response;

	stream = response = needle.get(source.url, { follow_max: 4 })
	.on("headers", function(headers) {
		// Some sources can be gunzipped twice (one for request, another for being a .txt.gz)
		stream = stream.pipe(gunzip()).pipe(gunzip());

		var filename = headers["content-disposition"] || url.parse(source.url).pathname;
		var detectedType = "generic";
        if (headers["content-type"] && headers["content-type"].match("xml")) detectedType = "xmlRss";
        if (filename.match(".txt.gz$")) detectedType = "dump";
     
		var type = status.type = importers[source.type] ? source.type : detectedType;

		// Pass on to the importer
		stream = importers[type](stream, source);<commit_msg>lib/importer: identation fix, explanation on streams<commit_after>/* Collect info hashes to queue for indexing
 */
 
var needle = require("needle");
var gunzip = require("gunzip-maybe");
var url = require("url");

var importers = { 
	dump: require("../importers/dump"),
	xmlRss: require("../importers/xml-rss"),
	generic: require("../importers/generic")
};

var db = require("../lib/db");

function collect(source, callback, onHash)
{
	var status = { found: 0, start: Date.now() }, stream, response;

	stream = response = needle.get(source.url, { follow_max: 4 })
	.on("headers", function(headers) {
		// Some sources can be gunzipped twice (one for request, another for being a .txt.gz)
		stream = stream.pipe(gunzip()).pipe(gunzip());

		var filename = headers["content-disposition"] || url.parse(source.url).pathname;
		var detectedType = "generic";
		if (headers["content-type"] && headers["content-type"].match("xml")) detectedType = "xmlRss";
		if (filename.match(".txt.gz$")) detectedType = "dump";

		// NOTE: if you prefer the full response rather than a stream, use in your importer:
		// var parts = []; stream.on("data", function(d) { parts.push(d) }); stream.on("end", function() { Buffer.concat(parts).toString() })
		// A stream is used because of the potential size of the import source

		var type = status.type = importers[source.type] ? source.type : detectedType;

		// Pass on to the importer
		stream = importers[type](stream, source);
{'fraction_non_alphanumeric': 0.17032755298651253, 'fraction_numerical': 0.0015414258188824663, 'mean_word_length': 4.265720081135902, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 8}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5851779', 'n_tokens_mistral': 836, 'n_tokens_neox': 786, 'n_words': 226}