<commit_before>import re
import logging
import asyncio
import aiohttp
import json

from functools import partialmethod

__author__ = "<NAME>"
__copyright__ = "Copyright 2015, <NAME>"
__license__ = "MIT"

API_URL = "https://api.telegram.org"
API_TIMEOUT = 60
RETRY_TIMEOUT = 30
RETRY_CODES = [429, 500, 502, 503, 504]
BOTAN_URL = "https://api.botan.io/track"

MESSAGE_TYPES = [
    "location", "photo", "document", "audio", "voice", "sticker", "contact"
]

logger = logging.getLogger("aiotg")

class TgSender(dict):
    def __repr__(self):
        uname = " (%s)" % self["username"] if "username" in self else ""
        return self['first_name'] + uname

class TgChat:
    def __init__(self, bot, message):
        self.bot = bot
        self.message = message
        self.sender = TgSender(message['from'])
        chat = message['chat']
        self.id = chat['id']
        self.type = chat['type']

    def send_text(self, text, **kwargs):
        return self.bot.send_message(self.id, text, **kwargs)

    def reply(self, text, markup=None):
        return self.send_text(text,
            reply_to_message_id=self.message["message_id"],
            disable_web_page_preview='true',
            reply_markup=json.dumps(markup)
        )

    def _send_to_chat(self, method, **options):
        return self.bot.api_call(
            method,
            chat_id=self.id,
            **options
        )

    send_audio = partialmethod(_send_to_chat, "sendAudio")
    send_photo = partialmethod(_send_to_chat, "sendPhoto")
    send_video = partialmethod(_send_to_chat, "sendVideo")
    send_document = partialmethod(_send_to_chat, "sendDocument")
    send_sticker = partialmethod(_send_to_chat, "sendSticker")
    send_voice = partialmethod(_send_to_chat, "sendVoice")
    send_locaton = partialmethod(_send_to_chat, "sendLocation")

    def forward_message(self, from_chat_id, message_id):
        return self.bot.api_call(
            "forwardMessage",
            chat_id=self.id,
            from_chat_id=from_chat_id,
            message_id=message_id
        )

    def is_group(self):
        return self.type == "group"


class TgBot:

    """Telegram bot framework designed for asyncio"""

    def __init__(self, api_token, api_timeout=API_TIMEOUT, botan_token=None):
        """
        api_token - Telegram bot token, ask @BotFather for this
        api_timeout (optional) - Timeout for long polling
        """
        self.api_token = api_token
        self.api_timeout = api_timeout
        self.botan_token = botan_token
        self.commands = []
        self._running = True

        self._default = lambda c, m: None

        def no_handle(mt):
            return lambda chat, msg: logger.debug("no handle for %s", mt)
        self._handlers = {mt: no_handle(mt) for mt in MESSAGE_TYPES}

    @asyncio.coroutine
    def api_call(self, method, **params):
        """Call Telegram API
        See https://core.telegram.org/bots/api for the reference
        """
        url = "{0}/bot{1}/{2}".format(API_URL, self.api_token, method)
        response = yield from aiohttp.post(url, data=params)

        if response.status == 200:
            return (yield from response.json())
        elif response.status in RETRY_CODES:
            logger.info("Server returned %d, retrying...", response.status)
            yield from response.release()
            yield from asyncio.sleep(RETRY_TIMEOUT)
            yield from self.api_call(method, **params)
        else:
            if response.headers['content-type'] == 'application/json':
                err_msg = (yield from response.json())["description"]
            else:
                err_msg = yield from response.read()
            logger.error(err_msg)
            raise RuntimeError(err_msg)

    _send_message = partialmethod(api_call, "sendMessage")

    def send_message(self, chat_id, text, **kwargs):
        """Send a text message to chat"""
        return self._send_message(chat_id=chat_id, text=text, **kwargs)

    def command(self, regexp):
        """Decorator for registering commands

        Example:
        @bot.command(r"/(start|help)")
        def usage(message, match):
            pass
        """
        def decorator(fn):
            self.commands.append((regexp, fn))
            return fn
        return decorator

    def default(self, callback):
        """Set callback for default command
        Default command is called on unrecognized commands for 1to1 chats
        """
        self._default = callback
        return callback

    def handle(self, msg_type):
        """Set handler for specific message type"""
        def wrap(callback):
            self._handlers[msg_type] = callback
            return callback
        return wrap

    @asyncio.coroutine
    def _track(self, message, name):
        response = yield from aiohttp.post(
            BOTAN_URL,
            params={
                "token": self.botan_token,
                "uid": message["from"]["id"],
                "name": name
            },
            data=json.dumps(message),
            headers={'content-type': 'application/json'}
        )
        if response.status != 200:
            logger.info("error submiting stats %d", response.status)
        yield from response.release()
        # res = yield from response.json()
        # if res["status"] != "accepted":
        #     logger.error("error submiting statistics %s: %s",
        #         res["status"], res.get("info", ""))

    def track(self, message, name="Message"):
        if self.botan_token:
            asyncio.async(self._track(message, name))

    @asyncio.coroutine
    def _process_message(self, message):
        chat = TgChat(self, message)

        for mt in MESSAGE_TYPES:
            if mt in message:
                self.track(message, mt)
                return self._handlers[mt](chat, message[mt])

        if "text" not in message:
            return

        text = message["text"].lower()

        for patterns, handler in self.commands:
            m = re.search(patterns, text)
            if m:
                self.track(message, handler.__name__)
                return handler(chat, m)

        # No match, run default if it's a 1to1 chat
        if not chat.is_group():
            self.track(message, "default")
            return self._default(chat, message)

    def stop(self):
        self._running = False

    @asyncio.coroutine
    def loop(self):
        """Return bot's main loop as coroutine

        Use it with asyncio:
        loop = asyncio.get_event_loop()
        loop.run_until_complete(bot.loop())

        or:
        loop.create_task(bot.loop())
        """
        offset = 0
        while self._running:
            resp = yield from self.api_call(
                'getUpdates',
                offset=offset + 1,
                timeout=self.api_timeout)

            for update in resp["result"]:
                logger.debug("update %s", update)
                offset = max(offset, update["update_id"])
                message = update["message"]
                asyncio.async(self._process_message(message))

    def run(self):
        loop = asyncio.get_event_loop()
        try:
            loop.run_until_complete(self.loop())
        except KeyboardInterrupt:
            self.stop()
<commit_msg>Display retry timeout in log<commit_after>import re
import logging
import asyncio
import aiohttp
import json

from functools import partialmethod

__author__ = "<NAME>"
__copyright__ = "Copyright 2015, <NAME>"
__license__ = "MIT"

API_URL = "https://api.telegram.org"
API_TIMEOUT = 60
RETRY_TIMEOUT = 30
RETRY_CODES = [429, 500, 502, 503, 504]
BOTAN_URL = "https://api.botan.io/track"

MESSAGE_TYPES = [
    "location", "photo", "document", "audio", "voice", "sticker", "contact"
]

logger = logging.getLogger("aiotg")

class TgSender(dict):
    def __repr__(self):
        uname = " (%s)" % self["username"] if "username" in self else ""
        return self['first_name'] + uname

class TgChat:
    def __init__(self, bot, message):
        self.bot = bot
        self.message = message
        self.sender = TgSender(message['from'])
        chat = message['chat']
        self.id = chat['id']
        self.type = chat['type']

    def send_text(self, text, **kwargs):
        return self.bot.send_message(self.id, text, **kwargs)

    def reply(self, text, markup=None):
        return self.send_text(text,
            reply_to_message_id=self.message["message_id"],
            disable_web_page_preview='true',
            reply_markup=json.dumps(markup)
        )

    def _send_to_chat(self, method, **options):
        return self.bot.api_call(
            method,
            chat_id=self.id,
            **options
        )

    send_audio = partialmethod(_send_to_chat, "sendAudio")
    send_photo = partialmethod(_send_to_chat, "sendPhoto")
    send_video = partialmethod(_send_to_chat, "sendVideo")
    send_document = partialmethod(_send_to_chat, "sendDocument")
    send_sticker = partialmethod(_send_to_chat, "sendSticker")
    send_voice = partialmethod(_send_to_chat, "sendVoice")
    send_locaton = partialmethod(_send_to_chat, "sendLocation")

    def forward_message(self, from_chat_id, message_id):
        return self.bot.api_call(
            "forwardMessage",
            chat_id=self.id,
            from_chat_id=from_chat_id,
            message_id=message_id
        )

    def is_group(self):
        return self.type == "group"


class TgBot:

    """Telegram bot framework designed for asyncio"""

    def __init__(self, api_token, api_timeout=API_TIMEOUT, botan_token=None):
        """
        api_token - Telegram bot token, ask @BotFather for this
        api_timeout (optional) - Timeout for long polling
        """
        self.api_token = api_token
        self.api_timeout = api_timeout
        self.botan_token = botan_token
        self.commands = []
        self._running = True

        self._default = lambda c, m: None

        def no_handle(mt):
            return lambda chat, msg: logger.debug("no handle for %s", mt)
        self._handlers = {mt: no_handle(mt) for mt in MESSAGE_TYPES}

    @asyncio.coroutine
    def api_call(self, method, **params):
        """Call Telegram API
        See https://core.telegram.org/bots/api for the reference
        """
        url = "{0}/bot{1}/{2}".format(API_URL, self.api_token, method)
        response = yield from aiohttp.post(url, data=params)

        if response.status == 200:
            return (yield from response.json())
        elif response.status in RETRY_CODES:
            logger.info("Server returned %d, retrying in %d sec.", response.status, RETRY_TIMEOUT)
            yield from response.release()
            yield from asyncio.sleep(RETRY_TIMEOUT)
            yield from self.api_call(method, **params)
        else:
            if response.headers['content-type'] == 'application/json':
                err_msg = (yield from response.json())["description"]
            else:
                err_msg = yield from response.read()
            logger.error(err_msg)
            raise RuntimeError(err_msg)

    _send_message = partialmethod(api_call, "sendMessage")

    def send_message(self, chat_id, text, **kwargs):
        """Send a text message to chat"""
        return self._send_message(chat_id=chat_id, text=text, **kwargs)

    def command(self, regexp):
        """Decorator for registering commands

        Example:
        @bot.command(r"/(start|help)")
        def usage(message, match):
            pass
        """
        def decorator(fn):
            self.commands.append((regexp, fn))
            return fn
        return decorator

    def default(self, callback):
        """Set callback for default command
        Default command is called on unrecognized commands for 1to1 chats
        """
        self._default = callback
        return callback

    def handle(self, msg_type):
        """Set handler for specific message type"""
        def wrap(callback):
            self._handlers[msg_type] = callback
            return callback
        return wrap

    @asyncio.coroutine
    def _track(self, message, name):
        response = yield from aiohttp.post(
            BOTAN_URL,
            params={
                "token": self.botan_token,
                "uid": message["from"]["id"],
                "name": name
            },
            data=json.dumps(message),
            headers={'content-type': 'application/json'}
        )
        if response.status != 200:
            logger.info("error submiting stats %d", response.status)
        yield from response.release()
        # res = yield from response.json()
        # if res["status"] != "accepted":
        #     logger.error("error submiting statistics %s: %s",
        #         res["status"], res.get("info", ""))

    def track(self, message, name="Message"):
        if self.botan_token:
            asyncio.async(self._track(message, name))

    @asyncio.coroutine
    def _process_message(self, message):
        chat = TgChat(self, message)

        for mt in MESSAGE_TYPES:
            if mt in message:
                self.track(message, mt)
                return self._handlers[mt](chat, message[mt])

        if "text" not in message:
            return

        text = message["text"].lower()

        for patterns, handler in self.commands:
            m = re.search(patterns, text)
            if m:
                self.track(message, handler.__name__)
                return handler(chat, m)

        # No match, run default if it's a 1to1 chat
        if not chat.is_group():
            self.track(message, "default")
            return self._default(chat, message)

    def stop(self):
        self._running = False

    @asyncio.coroutine
    def loop(self):
        """Return bot's main loop as coroutine

        Use it with asyncio:
        loop = asyncio.get_event_loop()
        loop.run_until_complete(bot.loop())

        or:
        loop.create_task(bot.loop())
        """
        offset = 0
        while self._running:
            resp = yield from self.api_call(
                'getUpdates',
                offset=offset + 1,
                timeout=self.api_timeout)

            for update in resp["result"]:
                logger.debug("update %s", update)
                offset = max(offset, update["update_id"])
                message = update["message"]
                asyncio.async(self._process_message(message))

    def run(self):
        loop = asyncio.get_event_loop()
        try:
            loop.run_until_complete(self.loop())
        except KeyboardInterrupt:
            self.stop()

{'fraction_non_alphanumeric': 0.11142935079105291, 'fraction_numerical': 0.005182760501909438, 'mean_word_length': 2.16670265601382, 'pattern_counts': {'":': 8, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 6, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6145779', 'n_tokens_mistral': 4351, 'n_tokens_neox': 4191, 'n_words': 1030}