<commit_before>import datetime
import logging
import os
import re
import time
import traceback

import requests
from bs4 import BeautifulSoup
from pymongo import MongoClient
from nltk.stem.wordnet import WordNetLemmatizer

from . import my_correction
from . import study_settings
from . import telegram
from . import train, remainder

env = os.getenv('BOT_ENV', 'staging')


class States:
    idle = 1
    langs_asked= 2


class App:
    def __init__(self, settings):
        dirname = os.path.dirname(os.path.realpath(__file__)) + '/'
        self.logger = logging.getLogger("bot")
        self.help_text = open(dirname + 'docs/help.txt').read()
        self.changelog_text = open(dirname + 'docs/changelog.txt').read()
        self.settings = settings
        self.wnl = WordNetLemmatizer()
        remainder.configure(settings)

        ###LOGGING
        access = logging.FileHandler('access.log')
        access.setLevel(logging.INFO)
        access.setFormatter(logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s'))

        error = logging.FileHandler('error.log')
        error.setLevel(logging.ERROR)
        error.setFormatter(logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s'))
        self.logger.addHandler(access)
        self.logger.addHandler(error)
        if env == 'debug':
            logging.basicConfig(level=logging.DEBUG)
        logging.warning("Cofiguration: %s" % (env,))

        self.db = MongoClient(settings.mongo['uri']).get_default_database()
        if 'users' not in self.db.collection_names():
            self.db.create_collection('users')
        self.users = self.db.users
        if 'remainders' not in self.db.collection_names():
            self.db.create_collection('remainders')
        remainder.recover_jobs()

        self.params['offset'] = 0
        logging.warning('Constructed')

    def listen(self):
        logging.warning('Listening')
        while True:
            self.get_updates()
            time.sleep(0.1)

            # app.run()

    def correct(self, string):
        baseurl_correction = 'http://service.afterthedeadline.com/checkDocument'
        correction = requests.get(baseurl_correction, {'data': string}).text
        correction = BeautifulSoup(correction, "lxml")

        if correction.find("option") is not None:
            string = correction.find("option").string
        return string

    def add_word(self, user, string):
        baseurl = 'https://translate.yandex.net/api/v1.5/tr.json/translate'
        # string = re.sub(r'[^A-Za-z\s]', '', string)
        # string = re.sub(r'\Wk+', ' ', string)
        string = string.lower()

        if len(string) == 0:
            telegram.send_message(user['chat_id'], "Wrong word")
            return

        # string = correct(string)
        string = my_correction.correct(string)
        t = time.time()
        if env != 'debug':
            string = self.wnl.lemmatize(string)
        print(time.time() - t, ' secs')
        string = string[0].upper() + string[1:]
        if 'direction' not in user:
            user['direction'] = 'en-ru'
        transtaltion = requests.get(baseurl, {
            'key': self.settings.translate_yandex['token'],
            'lang': user['direction'],
            'text': string
        })
        out_word = transtaltion.json()['text'][0]

        already_has = False
        for w in user['words']:
            already_has |= w["en"] == string
        if not already_has:
            user['words'].append({"en": string, "ru": out_word,
                                  "stage": study_settings.min_stage,
                                  "expiration_date": datetime.datetime.utcnow() + study_settings.stages[1],
                                  "creation_date": datetime.datetime.utcnow()})
            self.users.save(user)
            telegram.send_message(user['chat_id'], "Word added\n%s - %s" % (string, out_word))
        else:
            telegram.send_message(user['chat_id'], "Already exist!\n%s - %s" % (string, out_word))

    params = {}



    def get_list_word(self, user, text):
        str_out = "\n".join(["%s: (%s) %s - %s" % (i+1, w['stage'], w['en'], w['ru']) for i, w in zip(range(10**10),user['words'])])
        telegram.send_message(user['chat_id'], str_out)

    def start(self, user, text):
        telegram.send_message(user['chat_id'], """
Welcome!
I am a bot.
To learn how to use me, print /help.
Now you have to choose you foreign and native language.
Example: en-ru (en is foreign and ru is native)
To get list of language codes write help
        """)
        user['state'] = States.langs_asked
    def langs_ask(self, user, text):

        ans = requests.get('https://translate.yandex.net/api/v1.5/tr.json/getLangs',
                           {'key': self.settings.translate_yandex['token']})
        lang_list = ans.json()['dirs']
        if text not in lang_list:
            telegram.send_message(user['chat_id'], "Please, choose any of this:\n" + "\n".join(lang_list))
        else:
            telegram.send_message(user['chat_id'], "\"%s\" have successfully chosen" %(text, ))
            user['state'] = States.idle
            user['direction'] = text




    def help(self, user, text):
        telegram.send_message(user['chat_id'], self.help_text)

    def start_train(self, user, text):
        user['train']['type'] = 0
        train.do_train(user, text)

    def add_remainder(self, user, text):
        remainder.remove_job(user)
        tokens = text.split(' ')
        delta = datetime.timedelta()
        if len(tokens) >= 2:
            tokens = tokens[1].replace(' ', '').split(':')
            hours = int(tokens[0])
            minutes = int(tokens[1])
            delta = datetime.timedelta(hours=hours, minutes=minutes)
        remainder.add_job(user, datetime.datetime.utcnow() + delta)
        telegram.send_message(user['chat_id'], "Successfully set. Nearest at  %s" % (datetime.datetime.now() + delta,))

    def remove_remainder(self, user, text):
        remainder.remove_job(user)
        telegram.send_message(user['chat_id'], "Removed")

    def remove(self, user, text):
        if user['train']['type'] != 0:
            for w in user['words']:
                if w == user['train']['word']:
                    user['words'].remove(w)
                    str_out = "%s - %s" % (w['en'], w['ru'])
                    telegram.send_message(user['chat_id'], "Deleted:\n%s" %(str_out, ))

            train.do_train(user, text)


        else:
            tokens = text.split(" ")
            if len(tokens) > 1:
                cnt = int(tokens[1]) - 1
            else:
                cnt = -1
            str_out = "%s - %s" % (user['words'][cnt]['en'], user['words'][cnt]['ru'])
            del user['words'][cnt]
            telegram.send_message(user['chat_id'], "Word with index %s removed\n%s" % (cnt, str_out))

    comands = {
        'list': get_list_word,
        'rm': remove,
        'train': start_train,
        'end': train.end_train,
        'start': start,
        'help': help,
        'setremainder': add_remainder,<commit_msg>Bugfixes<commit_after>import datetime
import logging
import os
import re
import time
import traceback

import requests
from bs4 import BeautifulSoup
from pymongo import MongoClient
from nltk.stem.wordnet import WordNetLemmatizer

from . import my_correction
from . import study_settings
from . import telegram
from . import train, remainder

env = os.getenv('BOT_ENV', 'staging')


class States:
    idle = 1
    langs_asked = 2


class App:
    def __init__(self, settings):
        dirname = os.path.dirname(os.path.realpath(__file__)) + '/'
        self.logger = logging.getLogger("bot")
        self.help_text = open(dirname + 'docs/help.txt').read()
        self.changelog_text = open(dirname + 'docs/changelog.txt').read()
        self.settings = settings
        self.wnl = WordNetLemmatizer()
        remainder.configure(settings)

        ###LOGGING
        access = logging.FileHandler('access.log')
        access.setLevel(logging.INFO)
        access.setFormatter(logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s'))

        error = logging.FileHandler('error.log')
        error.setLevel(logging.ERROR)
        error.setFormatter(logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s'))
        self.logger.addHandler(access)
        self.logger.addHandler(error)
        if env == 'debug':
            logging.basicConfig(level=logging.DEBUG)
        logging.warning("Cofiguration: %s" % (env,))

        self.db = MongoClient(settings.mongo['uri']).get_default_database()
        if 'users' not in self.db.collection_names():
            self.db.create_collection('users')
        self.users = self.db.users
        if 'remainders' not in self.db.collection_names():
            self.db.create_collection('remainders')
        remainder.recover_jobs()

        self.params['offset'] = 0
        logging.warning('Constructed')

    def listen(self):
        logging.warning('Listening')
        while True:
            self.get_updates()
            time.sleep(0.1)

            # app.run()

    def correct(self, string):
        baseurl_correction = 'http://service.afterthedeadline.com/checkDocument'
        correction = requests.get(baseurl_correction, {'data': string}).text
        correction = BeautifulSoup(correction, "lxml")

        if correction.find("option") is not None:
            string = correction.find("option").string
        return string

    def add_word(self, user, string):
        baseurl = 'https://translate.yandex.net/api/v1.5/tr.json/translate'
        # string = re.sub(r'[^A-Za-z\s]', '', string)
        # string = re.sub(r'\Wk+', ' ', string)
        string = string.lower()

        if len(string) == 0:
            telegram.send_message(user['chat_id'], "Wrong word")
            return

        # string = correct(string)
        if user['foreign'] == 'en':
            string = my_correction.correct(string)
            if env != 'debug':
                string = self.wnl.lemmatize(string)
        string = string[0].upper() + string[1:]
        if 'direction' not in user:
            user['foreign'] = 'en'
            user['native'] = 'ru'
        direction = '%s-%s' % (user['foreign'], user['native'])
        transtaltion = requests.get(baseurl, {
            'key': self.settings.translate_yandex['token'],
            'lang': direction,
            'text': string
        })
        out_word = transtaltion.json()['text'][0]

        already_has = False
        for w in user['words']:
            already_has |= w["en"] == string
        if not already_has:
            user['words'].append({"en": string, "ru": out_word,
                                  "stage": study_settings.min_stage,
                                  "expiration_date": datetime.datetime.utcnow() + study_settings.stages[1],
                                  "creation_date": datetime.datetime.utcnow()})
            self.users.save(user)
            telegram.send_message(user['chat_id'], "Word added\n%s - %s" % (string, out_word))
        else:
            telegram.send_message(user['chat_id'], "Already exist!\n%s - %s" % (string, out_word))

    params = {}

    def get_list_word(self, user, text):
        str_out = "\n".join(["%s: (%s) %s - %s" % (i + 1, w['stage'], w['en'], w['ru']) for i, w in
                             zip(range(10 ** 10), user['words'])])
        telegram.send_message(user['chat_id'], str_out)

    def start(self, user, text):
        telegram.send_message(user['chat_id'], """
Welcome!
I am a bot.
To learn how to use me, print /help.
Now you have to choose you foreign and native language.
Example: en-ru (en is foreign and ru is native)
To get list of language codes write help
        """)
        user['state'] = States.langs_asked

    def langs_ask(self, user, text):
        ans = requests.get('https://translate.yandex.net/api/v1.5/tr.json/getLangs',
                           {'key': self.settings.translate_yandex['token']})
        lang_list = ans.json()['dirs']
        if text not in lang_list:
            telegram.send_message(user['chat_id'], "Please, choose any of this:\n" + "\n".join(lang_list))
        else:
            telegram.send_message(user['chat_id'], "\"%s\" have successfully chosen" % (text,))
            user['state'] = States.idle

            user['foreign'] = text[0:2]
            user['native'] = text[3:5]

    def help(self, user, text):
        telegram.send_message(user['chat_id'], self.help_text)

    def start_train(self, user, text):
        user['train']['type'] = 0
        train.do_train(user, text)

    def add_remainder(self, user, text):
        remainder.remove_job(user)
        tokens = text.split(' ')
        delta = datetime.timedelta()
        if len(tokens) >= 2:
            tokens = tokens[1].replace(' ', '').split(':')
            hours = int(tokens[0])
            minutes = int(tokens[1])
            delta = datetime.timedelta(hours=hours, minutes=minutes)
        remainder.add_job(user, datetime.datetime.utcnow() + delta)
        telegram.send_message(user['chat_id'], "Successfully set. Nearest at  %s" % (datetime.datetime.now() + delta,))

    def remove_remainder(self, user, text):
        remainder.remove_job(user)
        telegram.send_message(user['chat_id'], "Removed")

    def remove(self, user, text):
        if user['train']['type'] != 0:
            for w in user['words']:
                if w == user['train']['word']:
                    user['words'].remove(w)
                    str_out = "%s - %s" % (w['en'], w['ru'])
                    telegram.send_message(user['chat_id'], "Deleted:\n%s" % (str_out,))

            train.do_train(user, text)


        else:
            tokens = text.split(" ")
            if len(tokens) > 1:
                cnt = int(tokens[1]) - 1
            else:
                cnt = -1
            str_out = "%s - %s" % (user['words'][cnt]['en'], user['words'][cnt]['ru'])
            del user['words'][cnt]
            telegram.send_message(user['chat_id'], "Word with index %s removed\n%s" % (cnt, str_out))

    comands = {
        'list': get_list_word,
        'rm': remove,
        'train': start_train,
        'end': train.end_train,
        'start': start,
        'help': help,
        'setremainder': add_remainder,
{'fraction_non_alphanumeric': 0.150336782167905, 'fraction_numerical': 0.004444135823901118, 'mean_word_length': 2.2858772530230436, 'pattern_counts': {'":': 10, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 4, 'lorem ipsum': 0, 'www.': 0, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6386039', 'n_tokens_mistral': 4519, 'n_tokens_neox': 4414, 'n_words': 918}