<commit_before>__version__ = '0.1.0'

import re
import logging
from time import sleep, time
from logging.config import dictConfig
from concurrent.futures import ProcessPoolExecutor, ThreadPoolExecutor

import telegram

from utils.chatter_bot_api import ChatterBotFactory
from conf.log_conf import LOG_CONF
from conf import configurations as cfg

logger = logging.getLogger(__name__)


def get_updates(bot, update_id):
    last_update_id = 0
    updates = bot.getUpdates(offset=update_id)
    if updates:
        last_update_id = updates[-1].update_id

    for update in updates:
        text = update.message.text
        if text is None:
            continue
        for bot_address in cfg.BOT_ADDRESS_SET:
            if re.match(bot_address, text.lower(), re.IGNORECASE):
                update.message.text = re.sub(bot_address.lower(), '', text.lower(), re.IGNORECASE)
                return update, update.update_id + 1

    return [], last_update_id + 1


def process_messages(bot):
    update_id = None
    last_request = 0
    with ThreadPoolExecutor(3) as executor:
        while True:
            update, update_id = get_updates(bot, update_id)
            if update:
                message = update.message.text
                split_message = []

                # create all possible commands from message
                comb = ''
                for word in message.split():
                    comb += word + ' '
                    split_message.append(comb.strip().lower())<commit_msg>las_update_id is not resetting anymore<commit_after>__version__ = '0.1.0'

import re
import logging
from time import sleep, time
from logging.config import dictConfig
from concurrent.futures import ProcessPoolExecutor, ThreadPoolExecutor

import telegram
from telegram.error import TelegramError

from utils.chatter_bot_api import ChatterBotFactory
from conf.log_conf import LOG_CONF
from conf import configurations as cfg

logger = logging.getLogger(__name__)


def get_updates(bot, update_id):
    last_update_id = 0
    if update_id:
        last_update_id = update_id
    try:
        updates = bot.getUpdates(offset=update_id, timeout=2)
    except TelegramError:
        logging.exception('Could not get updates')
        return [], update_id

    if updates:
        last_update_id = updates[-1].update_id

    for update in updates:
        text = update.message.text
        if text is None:
            continue
        for bot_address in cfg.BOT_ADDRESS_SET:
            if re.match(bot_address, text.lower(), re.IGNORECASE):
                update.message.text = re.sub(bot_address.lower(), '', text.lower(), re.IGNORECASE)
                return update, update.update_id + 1

    return [], last_update_id


def process_messages(bot):
    update_id = None
    last_request = 0
    with ThreadPoolExecutor(3) as executor:
        while True:
            update, update_id = get_updates(bot, update_id)
            if update:
                message = update.message.text
                split_message = []

                # create all possible commands from message
                comb = ''
                for word in message.split():
                    comb += word + ' '
                    split_message.append(comb.strip().lower())
{'fraction_non_alphanumeric': 0.0773955773955774, 'fraction_numerical': 0.005528255528255528, 'mean_word_length': 2.378630705394191, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5534792', 'n_tokens_mistral': 923, 'n_tokens_neox': 888, 'n_words': 248}