<commit_before>/*
 * NTwitBot - main.js
 * Copyright (C) 2016-2017 <NAME>
 *
 * This program is free software: you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */

const timers        = require('timers');

const Data          = require('./data.js'),
      Generate      = require('./generate.js'),
      Process       = require('./process.js'),
      Retrieve      = require('./retrieve.js'),
      Twitter       = require('./twitter.js'),
      Util          = require('./util.js');

/**
 * Primary class of the bot. Handles all primary functions.
 */
module.exports = class Main {

    /**
     * Initialize bot modules.
     * @param {Object} secretData - The secret data for the Twitter API.
     * @param {Object} setup - The setup object from the config.
     */
    constructor(secretData, setup) {
        this.TWEET_INTERVAL = 15 /*min*/ * 60 /*s*/ * 1000 /*ms*/;

        this.secretData = secretData;
        this.setup = setup;

        this.utils = new Util();
        this.utils.log('Starting NTwitBot ' + process.env.npm_package_version + '..');
        this.utils.setDebug(this.setup.debug);

        this.dataHandler    = new Data(this.utils);
        this.processor      = new Process(this.utils);
        this.generator      = new Generate(this.utils);
        this.twitterHandler = new Twitter(this.secretData, this.utils);
        this.retriever      = new Retrieve(this.twitterHandler, this.utils);
    }

    /**
     * Initialize and setup bot.
     * @returns {Promise} Resolves when done setting up.
     */
    init() {
        return this.twitterHandler.verify().then((userID) => {
            this.utils.log('Verified Bot credentials, User ID is: ' + userID);

            return this.initState();
        }).then(() => {
            return this.dataHandler.createDataDir();

        }).catch((error) => {
            this.utils.logError('FATAL: Failed to initialize bot', error);
            throw new Error('Initializaion failure');
        });
    }

    /**
     * Initializes the state object.
     * @return {Promise} Resolves when done setting the state.
     */
    initState() {
        return this.dataHandler.readState().then((state) => {
            if (state === null) {
                this.state = {
                    trackedUsers: {}, // String dictionary with format 'userID : lastTweetID'
                    lastMention: 0    // Tweet ID of Last mention
                };
            } else {
                this.state = state;
            }
        });
    }

    /**
     * Starts the event handler.
     */
    start() {
        this.runUpdate();

        timers.setInterval(() => {
            this.runUpdate();
        }, this.TWEET_INTERVAL);
    }

    /**
     * Scheduled 15 min interval bot update.
     */
    runUpdate() {
        let updateState = false;

        this.utils.log('');
        this.utils.log('******************* Running update ******************* ');
        this.utils.log('');

        return this.handleTweets().then((newTweets) => {
            if (newTweets) {
                updateState = true;
            }

            return this.handleMentions();
        }).then((newMentions) => {
            if (newMentions) {
                updateState = true;
            }

            if (updateState) {
                return this.dataHandler.saveState(this.state);
            }
        }, (error) => {
            this.utils.logError('Failed to handle new mentions (skipping until next update)', error);

        }).then(() => {
            this.sendTweet();
        });
    }

    /**
     * Handles any new tweets from tracked users. Terminates bot upon save failure.
     * @return {Promise} Resolves with a boolean if new tweets were retrievd when done processing.
     */
    handleTweets() {
        return this.updateTracked().then(() => {
            return this.retriever.retrieveTweets(this.state.trackedUsers);

        }).then((retrievals) => {
            const tweets = this.processRetrievals(retrievals);

            if (tweets.length > 0) {
                this.utils.log('Retrieved tweets: ' + tweets.length + ' tweets to process');
                return this.dataHandler.saveTweetData(this.processor.processTweets(tweets));
            } else {
                this.utils.log('Retrieved tweets: No tweets to process');
                return Promise.reject();
            }
        }, (error) => {
            this.utils.logError('Failed to retrieve new tweets, skipping until next update', error);

        }).then(() => {
            return true;
        }, (error) => {
            if (error) {
                this.utils.logError('FATAL: Failed to save tweet data in database', error);
                throw new Error('Database error');
            }

            return false;
        });
    }

    /**
     * Combines tweet retrievals to a single array of tweets and updates the state.
     * @private
     * @param {2D Array} retrievals - An array of tweet retrievals (array).
     * @return {Array} The array of all received tweets.
     */
    processRetrievals(retrievals) {
        const tweets = [];

        for (const retrieval of retrievals) {
            if (retrieval.length > 0) {
                const firstTweet = retrieval[0];
                this.state.trackedUsers[firstTweet.user.id_str] = firstTweet.id_str;

                for (const tweet of retrieval) {
                    if (!tweet.hasOwnProperty('retweeted_status')) {<commit_msg>Edit to return false from handleTweets on retrieval failure<commit_after>/*
 * NTwitBot - main.js
 * Copyright (C) 2016-2017 <NAME>
 *
 * This program is free software: you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */

const timers        = require('timers');

const Data          = require('./data.js'),
      Generate      = require('./generate.js'),
      Process       = require('./process.js'),
      Retrieve      = require('./retrieve.js'),
      Twitter       = require('./twitter.js'),
      Util          = require('./util.js');

/**
 * Primary class of the bot. Handles all primary functions.
 */
module.exports = class Main {

    /**
     * Initialize bot modules.
     * @param {Object} secretData - The secret data for the Twitter API.
     * @param {Object} setup - The setup object from the config.
     */
    constructor(secretData, setup) {
        this.TWEET_INTERVAL = 15 /*min*/ * 60 /*s*/ * 1000 /*ms*/;

        this.secretData = secretData;
        this.setup = setup;

        this.utils = new Util();
        this.utils.log('Starting NTwitBot ' + process.env.npm_package_version + '..');
        this.utils.setDebug(this.setup.debug);

        this.dataHandler    = new Data(this.utils);
        this.processor      = new Process(this.utils);
        this.generator      = new Generate(this.utils);
        this.twitterHandler = new Twitter(this.secretData, this.utils);
        this.retriever      = new Retrieve(this.twitterHandler, this.utils);
    }

    /**
     * Initialize and setup bot.
     * @returns {Promise} Resolves when done setting up.
     */
    init() {
        return this.twitterHandler.verify().then((userID) => {
            this.utils.log('Verified Bot credentials, User ID is: ' + userID);

            return this.initState();
        }).then(() => {
            return this.dataHandler.createDataDir();

        }).catch((error) => {
            this.utils.logError('FATAL: Failed to initialize bot', error);
            throw new Error('Initializaion failure');
        });
    }

    /**
     * Initializes the state object.
     * @return {Promise} Resolves when done setting the state.
     */
    initState() {
        return this.dataHandler.readState().then((state) => {
            if (state === null) {
                this.state = {
                    trackedUsers: {}, // String dictionary with format 'userID : lastTweetID'
                    lastMention: 0    // Tweet ID of Last mention
                };
            } else {
                this.state = state;
            }
        });
    }

    /**
     * Starts the event handler.
     */
    start() {
        this.runUpdate();

        timers.setInterval(() => {
            this.runUpdate();
        }, this.TWEET_INTERVAL);
    }

    /**
     * Scheduled 15 min interval bot update.
     */
    runUpdate() {
        let updateState = false;

        this.utils.log('');
        this.utils.log('******************* Running update ******************* ');
        this.utils.log('');

        return this.handleTweets().then((newTweets) => {
            if (newTweets) {
                updateState = true;
            }

            return this.handleMentions();
        }).then((newMentions) => {
            if (newMentions) {
                updateState = true;
            }

            if (updateState) {
                return this.dataHandler.saveState(this.state);
            }
        }, (error) => {
            this.utils.logError('Failed to handle new mentions (skipping until next update)', error);

        }).then(() => {
            this.sendTweet();
        });
    }

    /**
     * Handles any new tweets from tracked users. Terminates bot upon save failure.
     * @return {Promise} Resolves with a boolean if new tweets were retrievd when done processing.
     */
    handleTweets() {
        return this.updateTracked().then(() => {
            return this.retriever.retrieveTweets(this.state.trackedUsers);

        }).then((retrievals) => {
            const tweets = this.processRetrievals(retrievals);

            if (tweets.length > 0) {
                this.utils.log('Retrieved tweets: ' + tweets.length + ' tweets to process');
                return this.dataHandler.saveTweetData(this.processor.processTweets(tweets));
            } else {
                this.utils.log('Retrieved tweets: No tweets to process');
                return Promise.reject();
            }
        }, (error) => {
            this.utils.logError('Failed to retrieve new tweets, skipping until next update', error);
            return Promise.reject();

        }).then(() => {
            return true;
        }, (error) => {
            if (error) {
                this.utils.logError('FATAL: Failed to save tweet data in database', error);
                throw new Error('Database error');
            }

            return false;
        });
    }

    /**
     * Combines tweet retrievals to a single array of tweets and updates the state.
     * @private
     * @param {2D Array} retrievals - An array of tweet retrievals (array).
     * @return {Array} The array of all received tweets.
     */
    processRetrievals(retrievals) {
        const tweets = [];

        for (const retrieval of retrievals) {
            if (retrieval.length > 0) {
                const firstTweet = retrieval[0];
                this.state.trackedUsers[firstTweet.user.id_str] = firstTweet.id_str;

                for (const tweet of retrieval) {
                    if (!tweet.hasOwnProperty('retweeted_status')) {
{'fraction_non_alphanumeric': 0.12523656710277298, 'fraction_numerical': 0.003949642063687978, 'mean_word_length': 1.8924321751546882, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 39, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4838249', 'n_tokens_mistral': 3378, 'n_tokens_neox': 3194, 'n_words': 934}