<commit_before># -*- coding: utf-8 -*-
from pokemongo_bot.event_manager import EventHandler
from pokemongo_bot.base_dir import _base_dir
import time
import telegram
import thread
import re
import pprint
from pokemongo_bot.datastore import Datastore
from telegram.utils import request
from chat_handler import ChatHandler

DEBUG_ON = False

class TelegramClass:

    update_id = None

    def __init__(self, bot, pokemons, config):
        self.bot = bot
        request.CON_POOL_SIZE = 16
        self.config = config
        self.chat_handler = ChatHandler(self.bot, pokemons)
        self.master = self.config.get('master')
        
        with self.bot.database as conn:
            # initialize the DB table if it does not exist yet
            initiator = TelegramDBInit(bot.database)
            if unicode(self.master).isnumeric(): # master is numeric
                self.bot.logger.info("Telegram master is valid (numeric): {}".format(self.master))
            elif self.master is not None: # Master is not numeric, fetch from db
                self.bot.logger.info("Telegram master is not numeric: {}".format(self.master))
                c = conn.cursor()
                srchmaster = re.sub(r'^@', '', self.master)
                c.execute("SELECT uid from telegram_uids where username = '{}'".format(srchmaster))
                results = c.fetchall()
                if len(results) > 0: # woohoo, we already saw a message from this master and therefore have a uid
                    self.bot.logger.info("Telegram master UID from datastore: {}".format(results[0][0]))
                    self.master = results[0][0]
                else: # uid not known yet
                    self.bot.logger.info("Telegram master UID not in datastore yet")

        self.pokemons = pokemons
        self._tbot = None
        self.config = config

    def connect(self):
        if DEBUG_ON: self.bot.logger.info("Not connected. Reconnecting")
        self._tbot = telegram.Bot(self.bot.config.telegram_token)
        try:
            self.update_id = self._tbot.getUpdates()[0].update_id
        except IndexError:
            self.update_id = None

    def grab_uid(self, update):
        with self.bot.database as conn:
            conn.execute("replace into telegram_uids (uid, username) values (?, ?)", (update.message.chat_id, update.message.from_user.username))
            conn.commit()
        self.master = update.message.chat_id

    def isMasterFromConfigFile(self, chat_id):
        if not hasattr(self, "master") or not self.master:
            return False
        if unicode(self.master).isnumeric():
            return unicode(chat_id) == unicode(self.master)
        else:
            with self.bot.database as conn:
                cur = conn.cursor()
                cur.execute("select username from telegram_uids where uid = ?", [chat_id])
                res = cur.fetchone()
                return res != None and unicode(res[0]) == unicode(re.replace(r'^@', '', self.master))

    def isMasterFromActiveLogins(self, chat_id):
        with self.bot.database as conn:
            cur = conn.cursor()
            cur.execute("select count(1) from telegram_logins where uid = ?", [chat_id])
            res = cur.fetchone()
            if res[0] == 1:
                return True
            else:
                return False

    def isAuthenticated(self, chat_id):
        return self.isMasterFromConfigFile(chat_id) or self.isMasterFromActiveLogins(chat_id)

    def deauthenticate(self, update):
        with self.bot.database as conn:
            cur = conn.cursor()
            cur.execute("delete from telegram_logins where uid = ?", [update.message.chat_id])
            conn.commit()
        self.chat_handler.sendMessage(chat_id=update.message.chat_id, parse_mode='Markdown', text="Logout completed")
        return

    def authenticate(self, update):
        (command, password) = update.message.text.split(' ')
        if password != self.config.get('password', None):
            self.chat_handler.sendMessage(chat_id=update.message.chat_id, parse_mode='Markdown', text="Invalid password")
        else:
            with self.bot.database as conn:
                cur = conn.cursor()
                cur.execute("delete from telegram_logins where uid = ?", [update.message.chat_id])
                cur.execute("insert into telegram_logins(uid) values(?)", [update.message.chat_id])
                conn.commit()
            self.chat_handler.sendMessage(chat_id=update.message.chat_id, parse_mode='Markdown', text="Authentication successful, you can now use all commands")
        return

    def run(self):            
        time.sleep(1)
        while True:
            if DEBUG_ON: self.bot.logger.info("Telegram loop running")
            if self._tbot is None:
                self.connect()
                
            for update in self._tbot.getUpdates(offset=self.update_id, timeout=10):
                self.update_id = update.update_id+1
                if update.message:
                    self.bot.logger.info("Telegram message from {} ({}): {}".format(update.message.from_user.username, update.message.from_user.id, update.message.text))
                    if update.message.text == "/start" or update.message.text == "/help":
                        res = (
                            "*Commands: *",
                            "/info - info about bot",
                            "/login <password> - authenticate with the bot; once authenticated, your ID will be registered with the bot and survive bot restarts",
                            "/logout - remove your ID from the 'authenticated' list",
                            "/sub <eventName> <parameters> - subscribe to eventName, with optional parameters, event name=all will subscribe to ALL events (LOTS of output!)",
                            "/unsub <eventName> <parameters> - unsubscribe from eventName; parameters must match the /sub parameters",
                            "/unsub everything - will remove all subscriptions for this uid",
                            "/showsubs - show current subscriptions",
                            "/events <filter> - show available events, filtered by regular expression  <filter>",
                            "/top <num> <cp-or-iv> - show top X pokemons, sorted by CP or IV",
                            "/evolved <num> <cp-or-iv> - show top x pokemon evolved, sorted by CP or IV",
                            "/hatched <num> <cp-or-iv> - show top x pokemon hatched, sorted by CP or IV",
                            "/caught <num> <cp-or-iv>- show top x pokemon caught, sorted by CP or IV",
                            "/pokestops - show last x pokestops visited",
                            "/released <num> <cp-or-iv> - show top x released, sorted by CP or IV",
                            "/vanished <num> <cp-or-iv> - show top x vanished, sorted by CP or IV",
                            "/softbans - info about possible softbans"
                        )
                        self.chat_handler.sendMessage(chat_id=update.message.chat_id, parse_mode='Markdown', text="\n".join(res))
                        continue
<commit_msg>Small change to let people know they can also sort by date<commit_after># -*- coding: utf-8 -*-
from pokemongo_bot.event_manager import EventHandler
from pokemongo_bot.base_dir import _base_dir
import time
import telegram
import thread
import re
import pprint
from pokemongo_bot.datastore import Datastore
from telegram.utils import request
from chat_handler import ChatHandler

DEBUG_ON = False

class TelegramClass:

    update_id = None

    def __init__(self, bot, pokemons, config):
        self.bot = bot
        request.CON_POOL_SIZE = 16
        self.config = config
        self.chat_handler = ChatHandler(self.bot, pokemons)
        self.master = self.config.get('master')
        
        with self.bot.database as conn:
            # initialize the DB table if it does not exist yet
            initiator = TelegramDBInit(bot.database)
            if unicode(self.master).isnumeric(): # master is numeric
                self.bot.logger.info("Telegram master is valid (numeric): {}".format(self.master))
            elif self.master is not None: # Master is not numeric, fetch from db
                self.bot.logger.info("Telegram master is not numeric: {}".format(self.master))
                c = conn.cursor()
                srchmaster = re.sub(r'^@', '', self.master)
                c.execute("SELECT uid from telegram_uids where username = '{}'".format(srchmaster))
                results = c.fetchall()
                if len(results) > 0: # woohoo, we already saw a message from this master and therefore have a uid
                    self.bot.logger.info("Telegram master UID from datastore: {}".format(results[0][0]))
                    self.master = results[0][0]
                else: # uid not known yet
                    self.bot.logger.info("Telegram master UID not in datastore yet")

        self.pokemons = pokemons
        self._tbot = None
        self.config = config

    def connect(self):
        if DEBUG_ON: self.bot.logger.info("Not connected. Reconnecting")
        self._tbot = telegram.Bot(self.bot.config.telegram_token)
        try:
            self.update_id = self._tbot.getUpdates()[0].update_id
        except IndexError:
            self.update_id = None

    def grab_uid(self, update):
        with self.bot.database as conn:
            conn.execute("replace into telegram_uids (uid, username) values (?, ?)", (update.message.chat_id, update.message.from_user.username))
            conn.commit()
        self.master = update.message.chat_id

    def isMasterFromConfigFile(self, chat_id):
        if not hasattr(self, "master") or not self.master:
            return False
        if unicode(self.master).isnumeric():
            return unicode(chat_id) == unicode(self.master)
        else:
            with self.bot.database as conn:
                cur = conn.cursor()
                cur.execute("select username from telegram_uids where uid = ?", [chat_id])
                res = cur.fetchone()
                return res != None and unicode(res[0]) == unicode(re.replace(r'^@', '', self.master))

    def isMasterFromActiveLogins(self, chat_id):
        with self.bot.database as conn:
            cur = conn.cursor()
            cur.execute("select count(1) from telegram_logins where uid = ?", [chat_id])
            res = cur.fetchone()
            if res[0] == 1:
                return True
            else:
                return False

    def isAuthenticated(self, chat_id):
        return self.isMasterFromConfigFile(chat_id) or self.isMasterFromActiveLogins(chat_id)

    def deauthenticate(self, update):
        with self.bot.database as conn:
            cur = conn.cursor()
            cur.execute("delete from telegram_logins where uid = ?", [update.message.chat_id])
            conn.commit()
        self.chat_handler.sendMessage(chat_id=update.message.chat_id, parse_mode='Markdown', text="Logout completed")
        return

    def authenticate(self, update):
        (command, password) = update.message.text.split(' ')
        if password != self.config.get('password', None):
            self.chat_handler.sendMessage(chat_id=update.message.chat_id, parse_mode='Markdown', text="Invalid password")
        else:
            with self.bot.database as conn:
                cur = conn.cursor()
                cur.execute("delete from telegram_logins where uid = ?", [update.message.chat_id])
                cur.execute("insert into telegram_logins(uid) values(?)", [update.message.chat_id])
                conn.commit()
            self.chat_handler.sendMessage(chat_id=update.message.chat_id, parse_mode='Markdown', text="Authentication successful, you can now use all commands")
        return

    def run(self):            
        time.sleep(1)
        while True:
            if DEBUG_ON: self.bot.logger.info("Telegram loop running")
            if self._tbot is None:
                self.connect()
                
            for update in self._tbot.getUpdates(offset=self.update_id, timeout=10):
                self.update_id = update.update_id+1
                if update.message:
                    self.bot.logger.info("Telegram message from {} ({}): {}".format(update.message.from_user.username, update.message.from_user.id, update.message.text))
                    if update.message.text == "/start" or update.message.text == "/help":
                        res = (
                            "*Commands: *",
                            "/info - info about bot",
                            "/login <password> - authenticate with the bot; once authenticated, your ID will be registered with the bot and survive bot restarts",
                            "/logout - remove your ID from the 'authenticated' list",
                            "/sub <eventName> <parameters> - subscribe to eventName, with optional parameters, event name=all will subscribe to ALL events (LOTS of output!)",
                            "/unsub <eventName> <parameters> - unsubscribe from eventName; parameters must match the /sub parameters",
                            "/unsub everything - will remove all subscriptions for this uid",
                            "/showsubs - show current subscriptions",
                            "/events <filter> - show available events, filtered by regular expression  <filter>",
                            "/top <num> <cp-or-iv-or-dated> - show top X pokemons, sorted by CP, IV, or Date",
                            "/evolved <num> <cp-or-iv-or-dated> - show top x pokemon evolved, sorted by CP, IV, or Date",
                            "/hatched <num> <cp-or-iv-or-dated> - show top x pokemon hatched, sorted by CP, IV, or Date",
                            "/caught <num> <cp-or-iv-or-dated> - show top x pokemon caught, sorted by CP, IV, or Date",
                            "/pokestops - show last x pokestops visited",
                            "/released <num> <cp-or-iv-or-dated> - show top x released, sorted by CP, IV, or Date",
                            "/vanished <num> <cp-or-iv-or-dated> - show top x vanished, sorted by CP, IV, or Date",
                            "/softbans - info about possible softbans"
                        )
                        self.chat_handler.sendMessage(chat_id=update.message.chat_id, parse_mode='Markdown', text="\n".join(res))
                        continue

{'fraction_non_alphanumeric': 0.10381763251853886, 'fraction_numerical': 0.0023345234825597363, 'mean_word_length': 1.940642035130224, 'pattern_counts': {'":': 2, '<': 41, '<?xml version=': 0, '>': 43, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1363165', 'n_tokens_mistral': 3984, 'n_tokens_neox': 3854, 'n_words': 1103}