<commit_before>var slug = require('slug')
    , FeedParser = require('feedparser')
    , Parse = require("parse").Parse
    , MongoClient = require('mongodb').MongoClient
    , assert = require('assert')
    , request = require('request')
    , _ = require('underscore');

var req = request('https://yts.re/rss')
    , feedparser = new FeedParser()
    , parsedMovies = []
    , PARSE_APP_KEY = process.env.PARSE_APP_KEY || ""
    , PARSE_JAVASCRIPT_KEY = process.env.PARSE_JAVASCRIPT_KEY || "";

var Movie = (function () {
    function Movie(item) {
        this.title = item.title;
        this.link = item.link;
        this.pub_date = item.pubDate;
        this.hash = slug(item.title, "_");
    }
    return Movie;
})();

function filterMovies(db, movies, callback){
    var slugs = _.pluck(movies, "hash");
    var collection = db.collection('movies');

    // Find already inserted movies
    collection.find({"hash": {$in:slugs}}).toArray(function(err, alreadyInsertedMovies) {
        assert.equal(err, null);

        // take the ones that are not included
        var alreadyInsertedSlugs = _.pluck(alreadyInsertedMovies, "hash");
        var filteredMovies = _.reject(movies, function(movie){
            return _.contains(alreadyInsertedSlugs, movie.hash);
        });

        callback(filteredMovies);
    });
}
function insertMovies(db, movies, callback){
    movies = movies || [];

    if(!movies.length){
        return callback(movies);
    }

    var collection = db.collection('movies');

    // Insert filtered movies
    collection.insert(movies, function(err, result) {
        assert.equal(err, null);
        console.log("Inserted "+ movies.length +" documents into the movies collection");
        callback(movies);
    });
}
function sendPushNotifications(movies){
    console.log("new movies:");
    console.log(movies);

    if(!movies.length){
        return;
    }

    if(PARSE_APP_KEY != "" && PARSE_JAVASCRIPT_KEY != ""){

        Parse.initialize(PARSE_APP_KEY, PARSE_JAVASCRIPT_KEY);
        var query = new Parse.Query(Parse.Installation);

        Parse.Push.send({
            where: query, // Set our Installation query
            data: {
                alert: "New movies uploaded: \n\n" + _.pluck(movies, "title").join("\n")
            }
        }, {
            success: function () {<commit_msg>sending push to the broadcast channel instead to everyone<commit_after>var slug = require('slug')
    , FeedParser = require('feedparser')
    , Parse = require("parse").Parse
    , MongoClient = require('mongodb').MongoClient
    , assert = require('assert')
    , request = require('request')
    , _ = require('underscore');

var req = request('https://yts.re/rss')
    , feedparser = new FeedParser()
    , parsedMovies = []
    , PARSE_APP_KEY = process.env.PARSE_APP_KEY || ""
    , PARSE_JAVASCRIPT_KEY = process.env.PARSE_JAVASCRIPT_KEY || "";

var Movie = (function () {
    function Movie(item) {
        this.title = item.title;
        this.link = item.link;
        this.pub_date = item.pubDate;
        this.hash = slug(item.title, "_");
    }
    return Movie;
})();

function filterMovies(db, movies, callback){
    var slugs = _.pluck(movies, "hash");
    var collection = db.collection('movies');

    // Find already inserted movies
    collection.find({"hash": {$in:slugs}}).toArray(function(err, alreadyInsertedMovies) {
        assert.equal(err, null);

        // take the ones that are not included
        var alreadyInsertedSlugs = _.pluck(alreadyInsertedMovies, "hash");
        var filteredMovies = _.reject(movies, function(movie){
            return _.contains(alreadyInsertedSlugs, movie.hash);
        });

        callback(filteredMovies);
    });
}
function insertMovies(db, movies, callback){
    movies = movies || [];

    if(!movies.length){
        return callback(movies);
    }

    var collection = db.collection('movies');

    // Insert filtered movies
    collection.insert(movies, function(err, result) {
        assert.equal(err, null);
        console.log("Inserted "+ movies.length +" documents into the movies collection");
        callback(movies);
    });
}
function sendPushNotifications(movies){
    console.log("new movies:");
    console.log(movies);

    if(!movies.length){
        return;
    }

    if(PARSE_APP_KEY != "" && PARSE_JAVASCRIPT_KEY != ""){

        Parse.initialize(PARSE_APP_KEY, PARSE_JAVASCRIPT_KEY);

        Parse.Push.send({
            channels: [""], // Set the broadcast channel
            data: {
                alert: "New movies uploaded: \n\n" + _.pluck(movies, "title").join("\n")
            }
        }, {
            success: function () {
{'fraction_non_alphanumeric': 0.1437125748502994, 'fraction_numerical': 0.0, 'mean_word_length': 2.7416, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3051636', 'n_tokens_mistral': 1465, 'n_tokens_neox': 1392, 'n_words': 281}