<commit_before>var byline = require("byline");
var _ = require("lodash");
var log = require("../lib/log");
var events = require("events");
var needle = require("needle");

module.exports = function(stream, source)
{
    var emitter = new events.EventEmitter();
    var i = 0, j = 0; // i - how many streams, j - how many completed

    stream.pipe(byline.createStream())
    .on("data", function(line)
    {
        var parts = line.toString().split("|"),
            infoHash = parts[0].toLowerCase(), cat = (parts[2] || "").toLowerCase(),
            additional = { }; // additional info found

        var infoUrl = parts[3];
        if (infoUrl.match("kat.cr|kickass.to")) additional.hints = { url: infoUrl }; // OR any torrent website that we know contains IMDB ID on it's info page
            
        // URL to torrent file
        if (parts[4].match(".torrent$")) additional.download = parts[4];

        // IMDB ID match
        var imdbMatch = (parts[5] && parts[5].match("(tt[0-9]+)")) || (parts[3] && parts[3].match("(tt[0-9]+)"));
        if (imdbMatch) additional.hints = { imdb_id: imdbMatch && imdbMatch[0] }; // no issue to override, because the hint url is for getting imdb_id anyway

        if (cat.match("porn") || cat.match("adult")) return; // exclude that for now

        //if (cat.match("video") && (cat.match("dvd") || cat.match("rip") || cat.match("tv")))
        if (cat.match("movie") || cat.match("dvd") || cat.match("rip") || cat.match("tv")/* || cat.match("video")*/) {
            additional.category = cat;
            hashReady(infoHash, _.extend(additional, source.addon || { }));
        };
    })
    .on("error", function(err) { log.error("dump", err) })
    .on("end", checkEnded());

    var checkingSeeders = source.minSeedersUrl && source.minSeeders;
    if (checkingSeeders) require("../lib/importer").getStream({ url: source.minSeedersUrl }, function(err, stream) {
        if (err) return checkingSeeders = false; // warning - bug - some info hashes will never be flushed 

        stream.pipe(byline.createStream())
        .on("data", function(line) {
            var parts = line.toString().split("|");
            var infoHash = parts[0].toLowerCase(), uploaders = parseInt(parts[1]), downloaders = parseInt(parts[2]);
            if (uploaders >= source.minSeeders) hashReady(infoHash, { uploaders: uploaders, downloaders: downloaders });
        })
        .on("end", checkEnded())
    });

    // TODO: make sure this is cleaned up<commit_msg>Allow passing category: true for some sources to disable category filter<commit_after>var byline = require("byline");
var _ = require("lodash");
var log = require("../lib/log");
var events = require("events");
var needle = require("needle");

module.exports = function(stream, source)
{
    var emitter = new events.EventEmitter();
    var i = 0, j = 0; // i - how many streams, j - how many completed

    stream.pipe(byline.createStream())
    .on("data", function(line)
    {
        var parts = line.toString().split("|"),
            infoHash = parts[0].toLowerCase(), cat = (parts[2] || "").toLowerCase(),
            additional = { }; // additional info found

        var infoUrl = parts[3];
        if (infoUrl.match("kat.cr|kickass.to")) additional.hints = { url: infoUrl }; // OR any torrent website that we know contains IMDB ID on it's info page
            
        // URL to torrent file
        if (parts[4].match(".torrent$")) additional.download = parts[4];

        // IMDB ID match
        var imdbMatch = (parts[5] && parts[5].match("(tt[0-9]+)")) || (parts[3] && parts[3].match("(tt[0-9]+)"));
        if (imdbMatch) additional.hints = { imdb_id: imdbMatch && imdbMatch[0] }; // no issue to override, because the hint url is for getting imdb_id anyway

        if (cat.match("porn") || cat.match("adult")) return; // exclude that for now

        // TODO: fluid category filter
        if (source.category===true || (cat.match("movie") || cat.match("dvd") || cat.match("rip") || cat.match("tv")/* || cat.match("video")*/)) {
            additional.category = cat;
            hashReady(infoHash, _.extend(additional, source.addon || { }));
        };
    })
    .on("error", function(err) { log.error("dump", err) })
    .on("end", checkEnded());

    var checkingSeeders = source.minSeedersUrl && source.minSeeders;
    if (checkingSeeders) require("../lib/importer").getStream({ url: source.minSeedersUrl }, function(err, stream) {
        if (err) return checkingSeeders = false; // warning - bug - some info hashes will never be flushed 

        stream.pipe(byline.createStream())
        .on("data", function(line) {
            var parts = line.toString().split("|");
            var infoHash = parts[0].toLowerCase(), uploaders = parseInt(parts[1]), downloaders = parseInt(parts[2]);
            if (uploaders >= source.minSeeders) hashReady(infoHash, { uploaders: uploaders, downloaders: downloaders });
        })
        .on("end", checkEnded())
    });

    // TODO: make sure this is cleaned up
{'fraction_non_alphanumeric': 0.17545255619653868, 'fraction_numerical': 0.007559180425701213, 'mean_word_length': 2.8886310904872388, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4637210', 'n_tokens_mistral': 1536, 'n_tokens_neox': 1517, 'n_words': 393}