<commit_before>'use strict';

var fs = require('fs'),
    moment = require('moment'),
    _ = require('lodash'),
    request = require('request').defaults({ encoding: null }),
    User = require('./user');

module.exports = function(config, bot, listener) {

    var logger = config.logger;

    var pokemon = JSON.parse(fs.readFileSync('./locale/pokemon.en.json'));
    var seen = []; // Contains encounter data of already processed pokemen

    logger.level = 'debug';

    // Webhook receives pokemon spawn info.
    // Unfiltered at this point
    listener.on('pokemon', function(payload) {

        // Webhook gave us an expired pokemon. It happens, just ignore
        if (moment().unix() > payload.disappear_time) {
            return;
        }

        // We have seen this pokemon
        if (_.find(seen, { disappear: payload.disappear_time}) !== undefined) {
            return;
        }

        seen.push({
            id: payload.encounter_id,
            disappear: payload.disappear_time
        });

        // Find all users that are active and watching this pokemon
        User.find({ active: true, watchlist: Number(payload.pokemon_id) })
            .then(function(users) {
                logger.info(
                    'Wild %s appeared!\t Disappear time %s\t Users watching %s\t Seen pokemon %s',
                    pokemon[payload.pokemon_id],
                    payload.disappear_time,
                    _.map(users, 'telegramId'),
                    seen.length
                );
<commit_msg>Ignore duplicates by encounter id<commit_after>'use strict';

var fs = require('fs'),
    moment = require('moment'),
    _ = require('lodash'),
    request = require('request').defaults({ encoding: null }),
    User = require('./user');

module.exports = function(config, bot, listener) {

    var logger = config.logger;

    var pokemon = JSON.parse(fs.readFileSync('./locale/pokemon.en.json'));
    var seen = []; // Contains encounter data of already processed pokemen

    logger.level = 'debug';

    // Webhook receives pokemon spawn info.
    // Unfiltered at this point
    listener.on('pokemon', function(payload) {

        // Webhook gave us an expired pokemon. It happens, just ignore
        if (moment().unix() > payload.disappear_time) {
            return;
        }

        // We have seen this pokemon
        if (_.find(seen, { id: payload.encounter_id}) !== undefined) {
            logger.debug(
                'Ignoring duplicate encounter of %s\t encounter_id %s',
                pokemon[payload.pokemon_id],
                payload.encounter_id
            );
            return;
        }

        seen.push({
            id: payload.encounter_id,
            disappear: payload.disappear_time
        });

        // Find all users that are active and watching this pokemon
        User.find({ active: true, watchlist: Number(payload.pokemon_id) })
            .then(function(users) {
                logger.info(
                    'Wild %s appeared!\t Disappear time %s\t Users watching %s\t Seen pokemon %s',
                    pokemon[payload.pokemon_id],
                    payload.disappear_time,
                    _.map(users, 'telegramId'),
                    seen.length
                );

{'fraction_non_alphanumeric': 0.11788120024494794, 'fraction_numerical': 0.0, 'mean_word_length': 1.9169642857142857, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2236957', 'n_tokens_mistral': 915, 'n_tokens_neox': 880, 'n_words': 234}