<commit_before>var async = require("async");
var namedQueue = require("named-queue");
var qs = require("querystring");
var _ = require("lodash");
var needle = require("needle");
var Stremio = require("stremio-addons");
var helpers = require("./helpers.js")

// Use cinemeta to pull names of movies and series against IMDB IDs
var stremio = new Stremio.Client();
stremio.add("http://cinemeta.strem.io/stremioget/stremio/v1");
//stremio.add("http://localhost:3005/stremioget/stremio/v1");

// Constants
var CACHE_TTL = 4*60*60*1000; // if we don't find an item, how long does it stay in the cache as "not found" before we retry it 
var MAX_CACHE_SIZE = 20000;

var GOOGLE_AJAX_API = "http://ajax.googleapis.com/ajax/services/search/web?v=1.0&rsz=large&q=";
var GOOGLE_SEARCH = "https://www.google.com/search?safe=off&site=&source=hp&q=";

function assert(condition, log)
{ 
    if (!condition) console.log("name-retriever: "+log);
}

// Index entry in our in-mem index
function indexEntry(entry) {
    if (entry.year) entry.year = parseInt(entry.year.toString().split("-")[0]); // first year for series
    var n = helpers.simplifyName(entry);
    if (!meta[n]) meta[n] = [];
    meta[n].push(entry);
    byImdb[entry.imdb_id] = entry;
}

// Find in our metadata set
var pulled = { movie: false, series: false };
var meta = { }, byImdb = { };

function metadataFind(query, cb) {
    // It's OK if we don't pass type and we don't fetch names, because we will go to google fallback
    if (query.type && !pulled[query.type]) stremio.call("names."+query.type, { }, function(err, res) {
        if (err) console.error(err);
        if (res) {
            res.forEach(indexEntry);
            pulled[query.type] = 1;
        }
        match();
    });
    else process.nextTick(match);

    function match() {
        var matches = meta[query.name] || [ ];
        var m = matches.find(function(match) {
            if (! match.type === query.type) return false

            if (query.type === 'movie' && query.hasOwnProperty('year')) 
                return helpers.yearSimilar(query.year, match.year)
           
            return true
        })
        return cb(null, m && m.imdb_id);
    };
}

function imdbFind(task, cb, simpler) {

    function nextTick() {
        if (!simpler && task.year)
            imdbFind(task, cb, true)
        else
            webFind(task, cb)
    }

    function matchSimilar(parsed, callback) {

        var pick
        var secondBest
        var firstResult

        var similarityGoal = 0.6

        parsed.some(elm => {
            if (!task.type || (task.type == 'movie' && elm.q == 'feature') || (task.type == 'series' && (elm.q == 'TV series' || elm.q == 'TV mini-series'))) {

                if (helpers.yearSimilar(task.year, elm.y)) {

                    // try to match by levenshtein distance
                    var similarity = helpers.nameSimilar(task.name, elm.l)

                    if (similarity > similarityGoal) {
                        if (!pick || (pick && similarity > pick.similarity)) {
                            pick = elm
                            pick.similarity = similarity
                        }
                    }

                    // fallback to non-levenshtein distance logic
                    if (!secondBest && helpers.nameAlmostSimilar(task.name, elm.l))
                        secondBest = elm

                    // if nothing else is found, pick first result
                    // (because what we're searching for might be the alternative name of the first result)
                    // this is ignored if strict mode enabled
                    if (!firstResult && !task.strict)
                        firstResult = elm
                }

            }
        })

        callback(pick || secondBest || firstResult || null)
    }

    // we first search imdb for name + year (if we have a year set)
    // if it fails we search by name only (simpler)
    var searchTerm = !simpler && task.year ? task.name + ' ' + task.year : task.name
    
    var imdbSearchUrl = 'http://sg.media-imdb.com/suggests/' + searchTerm.charAt(0).toLowerCase() + '/' + encodeURIComponent(searchTerm)  + '.json'

    needle.get(imdbSearchUrl, function(err, res) {
        if (!err && res.statusCode == 200 && res.body) {

            res.body = String.fromCharCode.apply(null, res.body)

            var imdbParse = JSON.parse(res.body.match(/{.*}/g))

            if (imdbParse.d) {

                var selected

                matchSimilar(imdbParse.d, function(selected) {
                    if (selected)
                        cb(null, selected.id, { match: imdbSearchUrl })
                    else nextTick()
                })

            } else nextTick()
        } else nextTick()
    })
}

// Find in the web / Google
function webFind(task, cb) {

    if (task.strict || task.noGoogle) return cb(null, null, null) // strict and noGoogle don't search google

    var opts = {
        follow_max: 3,
        open_timeout: 15*1000,
        headers: { "user-agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/46.0.2490.86 Safari/537.36" }
    };

    if (task.hintUrl) return needle.get(task.hintUrl, opts, function(err, resp, body) {
        if (err) return cb(err);
        var match = body && body.match(new RegExp("\/title\/(tt[0-9]+)")); // Match IMDB Id from the whole body
        var id = match && match[1];
        cb(null, id, { match: task.hintUrl });
    });

    // WARNING: www. vs not?  is there difference?
    // no quotes - they can actually make the results dumber
    var query = "site:imdb.com "
        +task.name.toLowerCase()+(task.year ? " "+task.year : "")
        +((task.type=="series") ? " \"tv series\"" : ""); // Compute that now so that we can use the mapping

    // Google search api is deprecated, use this
    webFind({ hintUrl: GOOGLE_SEARCH+encodeURIComponent(query) }, cb);
}

// In-memory cache for matched items, to avoid flooding Google (or whatever search api we use)
var cache = { };

// Outside API
function nameToImdb(args, cb) {
    args = typeof(args)=="string" ? { name: args } : args;

    if (!args.hasOwnProperty('noGoogle')) args.noGoogle = true
    
    var q = _.pick(args, "name", "year", "type");
    q.name = helpers.simplifyName(q);

    if (! q.name) return cb(new Error("empty name"));

    if (q.year && typeof(q.year)=="string") q.year = parseInt(q.year.split("-")[0]);
    if (q.year && isNaN(q.year)) return cb(new Error("invalid year"));

    if (q.type && !(q.type=="movie" || q.type=="series")) return cb(null, null); // no match for other types

    var hash = new Buffer(args.hintUrl || _.values(q).join(":")).toString("ascii"); // convert to ASCII since EventEmitter bugs with UTF8
    if (cache.hasOwnProperty(hash)) return cb(null, cache[hash]);

    queue.push({ id: hash, q: q, args: args }, function(err, imdb_id, match) {
        if (err) return cb(err);
        if (imdb_id) {
            cache[hash] = imdb_id;
            setTimeout(function() { delete cache[hash] }, CACHE_TTL);
        }
        cb(null, imdb_id, match);
    });
};

var queue = new namedQueue(function(task, cb) {
    // Find it in our metadata, if not, fallback to IMDB API, then Google
    metadataFind(task.q, function(err, id) {
        if (err) return cb(err);
        if (id) return cb(null, id, { match: "metadata" });
        imdbFind(task.args, cb);
    });
}, 3);


module.exports = nameToImdb;
module.exports.byImdb = byImdb;
<commit_msg>Name Should Be Simplified in Args Too<commit_after>var async = require("async");
var namedQueue = require("named-queue");
var qs = require("querystring");
var _ = require("lodash");
var needle = require("needle");
var Stremio = require("stremio-addons");
var helpers = require("./helpers.js")

// Use cinemeta to pull names of movies and series against IMDB IDs
var stremio = new Stremio.Client();
stremio.add("http://cinemeta.strem.io/stremioget/stremio/v1");
//stremio.add("http://localhost:3005/stremioget/stremio/v1");

// Constants
var CACHE_TTL = 4*60*60*1000; // if we don't find an item, how long does it stay in the cache as "not found" before we retry it 
var MAX_CACHE_SIZE = 20000;

var GOOGLE_AJAX_API = "http://ajax.googleapis.com/ajax/services/search/web?v=1.0&rsz=large&q=";
var GOOGLE_SEARCH = "https://www.google.com/search?safe=off&site=&source=hp&q=";

function assert(condition, log)
{ 
    if (!condition) console.log("name-retriever: "+log);
}

// Index entry in our in-mem index
function indexEntry(entry) {
    if (entry.year) entry.year = parseInt(entry.year.toString().split("-")[0]); // first year for series
    var n = helpers.simplifyName(entry);
    if (!meta[n]) meta[n] = [];
    meta[n].push(entry);
    byImdb[entry.imdb_id] = entry;
}

// Find in our metadata set
var pulled = { movie: false, series: false };
var meta = { }, byImdb = { };

function metadataFind(query, cb) {
    // It's OK if we don't pass type and we don't fetch names, because we will go to google fallback
    if (query.type && !pulled[query.type]) stremio.call("names."+query.type, { }, function(err, res) {
        if (err) console.error(err);
        if (res) {
            res.forEach(indexEntry);
            pulled[query.type] = 1;
        }
        match();
    });
    else process.nextTick(match);

    function match() {
        var matches = meta[query.name] || [ ];
        var m = matches.find(function(match) {
            if (! match.type === query.type) return false

            if (query.type === 'movie' && query.hasOwnProperty('year')) 
                return helpers.yearSimilar(query.year, match.year)
           
            return true
        })
        return cb(null, m && m.imdb_id);
    };
}

function imdbFind(task, cb, simpler) {

    function nextTick() {
        if (!simpler && task.year)
            imdbFind(task, cb, true)
        else
            webFind(task, cb)
    }

    function matchSimilar(parsed, callback) {

        var pick
        var secondBest
        var firstResult

        var similarityGoal = 0.6

        parsed.some(elm => {
            if (!task.type || (task.type == 'movie' && elm.q == 'feature') || (task.type == 'series' && (elm.q == 'TV series' || elm.q == 'TV mini-series'))) {

                if (helpers.yearSimilar(task.year, elm.y)) {

                    // try to match by levenshtein distance
                    var similarity = helpers.nameSimilar(task.name, elm.l)

                    if (similarity > similarityGoal) {
                        if (!pick || (pick && similarity > pick.similarity)) {
                            pick = elm
                            pick.similarity = similarity
                        }
                    }

                    // fallback to non-levenshtein distance logic
                    if (!secondBest && helpers.nameAlmostSimilar(task.name, elm.l))
                        secondBest = elm

                    // if nothing else is found, pick first result
                    // (because what we're searching for might be the alternative name of the first result)
                    // this is ignored if strict mode enabled
                    if (!firstResult && !task.strict)
                        firstResult = elm
                }

            }
        })

        callback(pick || secondBest || firstResult || null)
    }

    // we first search imdb for name + year (if we have a year set)
    // if it fails we search by name only (simpler)
    var searchTerm = !simpler && task.year ? task.name + ' ' + task.year : task.name
    
    var imdbSearchUrl = 'http://sg.media-imdb.com/suggests/' + searchTerm.charAt(0).toLowerCase() + '/' + encodeURIComponent(searchTerm)  + '.json'

    needle.get(imdbSearchUrl, function(err, res) {
        if (!err && res.statusCode == 200 && res.body) {

            res.body = String.fromCharCode.apply(null, res.body)

            var imdbParse = JSON.parse(res.body.match(/{.*}/g))

            if (imdbParse.d) {

                var selected

                matchSimilar(imdbParse.d, function(selected) {
                    if (selected)
                        cb(null, selected.id, { match: imdbSearchUrl })
                    else nextTick()
                })

            } else nextTick()
        } else nextTick()
    })
}

// Find in the web / Google
function webFind(task, cb) {

    if (task.strict || task.noGoogle) return cb(null, null, null) // strict and noGoogle don't search google

    var opts = {
        follow_max: 3,
        open_timeout: 15*1000,
        headers: { "user-agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/46.0.2490.86 Safari/537.36" }
    };

    if (task.hintUrl) return needle.get(task.hintUrl, opts, function(err, resp, body) {
        if (err) return cb(err);
        var match = body && body.match(new RegExp("\/title\/(tt[0-9]+)")); // Match IMDB Id from the whole body
        var id = match && match[1];
        cb(null, id, { match: task.hintUrl });
    });

    // WARNING: www. vs not?  is there difference?
    // no quotes - they can actually make the results dumber
    var query = "site:imdb.com "
        +task.name.toLowerCase()+(task.year ? " "+task.year : "")
        +((task.type=="series") ? " \"tv series\"" : ""); // Compute that now so that we can use the mapping

    // Google search api is deprecated, use this
    webFind({ hintUrl: GOOGLE_SEARCH+encodeURIComponent(query) }, cb);
}

// In-memory cache for matched items, to avoid flooding Google (or whatever search api we use)
var cache = { };

// Outside API
function nameToImdb(args, cb) {
    args = typeof(args)=="string" ? { name: args } : args;

    if (!args.hasOwnProperty('noGoogle')) args.noGoogle = true

    args.name = helpers.simplifyName(args);
    
    var q = _.pick(args, "name", "year", "type");

    if (! q.name) return cb(new Error("empty name"));

    if (q.year && typeof(q.year)=="string") q.year = parseInt(q.year.split("-")[0]);
    if (q.year && isNaN(q.year)) return cb(new Error("invalid year"));

    if (q.type && !(q.type=="movie" || q.type=="series")) return cb(null, null); // no match for other types

    var hash = new Buffer(args.hintUrl || _.values(q).join(":")).toString("ascii"); // convert to ASCII since EventEmitter bugs with UTF8
    if (cache.hasOwnProperty(hash)) return cb(null, cache[hash]);

    queue.push({ id: hash, q: q, args: args }, function(err, imdb_id, match) {
        if (err) return cb(err);
        if (imdb_id) {
            cache[hash] = imdb_id;
            setTimeout(function() { delete cache[hash] }, CACHE_TTL);
        }
        cb(null, imdb_id, match);
    });
};

var queue = new namedQueue(function(task, cb) {
    // Find it in our metadata, if not, fallback to IMDB API, then Google
    metadataFind(task.q, function(err, id) {
        if (err) return cb(err);
        if (id) return cb(null, id, { match: "metadata" });
        imdbFind(task.args, cb);
    });
}, 3);


module.exports = nameToImdb;
module.exports.byImdb = byImdb;

{'fraction_non_alphanumeric': 0.14490835704360486, 'fraction_numerical': 0.009131211539734004, 'mean_word_length': 2.2419562419562418, 'pattern_counts': {'":': 4, '<': 3, '<?xml version=': 0, '>': 9, 'https://': 2, 'lorem ipsum': 0, 'www.': 4, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '28366', 'n_tokens_mistral': 4888, 'n_tokens_neox': 4651, 'n_words': 1278}