<commit_before>"""
Utilities for working with the Telegram API itself (such as handy methods
to convert between an entity like a User, Chat, etc. into its Input version)
"""
import base64
import binascii
import imghdr
import inspect
import io
import itertools
import logging
import math
import mimetypes
import os
import pathlib
import re
import struct
from collections import namedtuple
from mimetypes import guess_extension
from types import GeneratorType

from .extensions import markdown, html
from .helpers import add_surrogate, del_surrogate, strip_text
from .tl import types

try:
    import hachoir
    import hachoir.metadata
    import hachoir.parser
except ImportError:
    hachoir = None

# Register some of the most common mime-types to avoid any issues.
# See https://github.com/LonamiWebs/Telethon/issues/1096.
mimetypes.add_type('image/png', '.png')
mimetypes.add_type('image/jpeg', '.jpeg')
mimetypes.add_type('image/webp', '.webp')
mimetypes.add_type('image/gif', '.gif')
mimetypes.add_type('image/bmp', '.bmp')
mimetypes.add_type('image/x-tga', '.tga')
mimetypes.add_type('image/tiff', '.tiff')
mimetypes.add_type('image/vnd.adobe.photoshop', '.psd')

mimetypes.add_type('video/mp4', '.mp4')
mimetypes.add_type('video/quicktime', '.mov')
mimetypes.add_type('video/avi', '.avi')

mimetypes.add_type('audio/mpeg', '.mp3')
mimetypes.add_type('audio/m4a', '.m4a')
mimetypes.add_type('audio/aac', '.aac')
mimetypes.add_type('audio/ogg', '.ogg')
mimetypes.add_type('audio/flac', '.flac')

mimetypes.add_type('application/x-tgsticker', '.tgs')

USERNAME_RE = re.compile(
    r'@|(?:https?://)?(?:www\.)?(?:telegram\.(?:me|dog)|t\.me)/(@|joinchat/)?'
)
TG_JOIN_RE = re.compile(
    r'tg://(join)\?invite='
)

# The only shorter-than-five-characters usernames are those used for some
# special, very well known bots. This list may be incomplete though:
#    "[...] @gif, @vid, @pic, @bing, @wiki, @imdb and @bold [...]"
#
# See https://telegram.org/blog/inline-bots#how-does-it-work
VALID_USERNAME_RE = re.compile(
    r'^([a-z](?:(?!__)\w){3,30}[a-z\d]'
    r'|gif|vid|pic|bing|wiki|imdb|bold|vote|like|coub)$',
    re.IGNORECASE
)

_FileInfo = namedtuple('FileInfo', 'dc_id location size')

_log = logging.getLogger(__name__)


def chunks(iterable, size=100):
    """
    Turns the given iterable into chunks of the specified size,
    which is 100 by default since that's what Telegram uses the most.
    """
    it = iter(iterable)
    size -= 1
    for head in it:
        yield itertools.chain([head], itertools.islice(it, size))


def get_display_name(entity):
    """
    Gets the display name for the given :tl:`User`,
    :tl:`Chat` or :tl:`Channel`. Returns an empty string otherwise.
    """
    if isinstance(entity, types.User):
        if entity.last_name and entity.first_name:
            return '{} {}'.format(entity.first_name, entity.last_name)
        elif entity.first_name:
            return entity.first_name
        elif entity.last_name:
            return entity.last_name
        else:
            return ''

    elif isinstance(entity, (types.Chat, types.Channel)):
        return entity.title

    return ''


def get_extension(media):
    """Gets the corresponding extension for any Telegram media."""

    # Photos are always compressed as .jpg by Telegram<commit_msg>Fix get_display_name not handling ChatForbidden (#1617)<commit_after>"""
Utilities for working with the Telegram API itself (such as handy methods
to convert between an entity like a User, Chat, etc. into its Input version)
"""
import base64
import binascii
import imghdr
import inspect
import io
import itertools
import logging
import math
import mimetypes
import os
import pathlib
import re
import struct
from collections import namedtuple
from mimetypes import guess_extension
from types import GeneratorType

from .extensions import markdown, html
from .helpers import add_surrogate, del_surrogate, strip_text
from .tl import types

try:
    import hachoir
    import hachoir.metadata
    import hachoir.parser
except ImportError:
    hachoir = None

# Register some of the most common mime-types to avoid any issues.
# See https://github.com/LonamiWebs/Telethon/issues/1096.
mimetypes.add_type('image/png', '.png')
mimetypes.add_type('image/jpeg', '.jpeg')
mimetypes.add_type('image/webp', '.webp')
mimetypes.add_type('image/gif', '.gif')
mimetypes.add_type('image/bmp', '.bmp')
mimetypes.add_type('image/x-tga', '.tga')
mimetypes.add_type('image/tiff', '.tiff')
mimetypes.add_type('image/vnd.adobe.photoshop', '.psd')

mimetypes.add_type('video/mp4', '.mp4')
mimetypes.add_type('video/quicktime', '.mov')
mimetypes.add_type('video/avi', '.avi')

mimetypes.add_type('audio/mpeg', '.mp3')
mimetypes.add_type('audio/m4a', '.m4a')
mimetypes.add_type('audio/aac', '.aac')
mimetypes.add_type('audio/ogg', '.ogg')
mimetypes.add_type('audio/flac', '.flac')

mimetypes.add_type('application/x-tgsticker', '.tgs')

USERNAME_RE = re.compile(
    r'@|(?:https?://)?(?:www\.)?(?:telegram\.(?:me|dog)|t\.me)/(@|joinchat/)?'
)
TG_JOIN_RE = re.compile(
    r'tg://(join)\?invite='
)

# The only shorter-than-five-characters usernames are those used for some
# special, very well known bots. This list may be incomplete though:
#    "[...] @gif, @vid, @pic, @bing, @wiki, @imdb and @bold [...]"
#
# See https://telegram.org/blog/inline-bots#how-does-it-work
VALID_USERNAME_RE = re.compile(
    r'^([a-z](?:(?!__)\w){3,30}[a-z\d]'
    r'|gif|vid|pic|bing|wiki|imdb|bold|vote|like|coub)$',
    re.IGNORECASE
)

_FileInfo = namedtuple('FileInfo', 'dc_id location size')

_log = logging.getLogger(__name__)


def chunks(iterable, size=100):
    """
    Turns the given iterable into chunks of the specified size,
    which is 100 by default since that's what Telegram uses the most.
    """
    it = iter(iterable)
    size -= 1
    for head in it:
        yield itertools.chain([head], itertools.islice(it, size))


def get_display_name(entity):
    """
    Gets the display name for the given :tl:`User`,
    :tl:`Chat` or :tl:`Channel`. Returns an empty string otherwise.
    """
    if isinstance(entity, types.User):
        if entity.last_name and entity.first_name:
            return '{} {}'.format(entity.first_name, entity.last_name)
        elif entity.first_name:
            return entity.first_name
        elif entity.last_name:
            return entity.last_name
        else:
            return ''

    elif isinstance(entity, (types.Chat, types.ChatForbidden, types.Channel)):
        return entity.title

    return ''


def get_extension(media):
    """Gets the corresponding extension for any Telegram media."""

    # Photos are always compressed as .jpg by Telegram
{'fraction_non_alphanumeric': 0.14405385190725506, 'fraction_numerical': 0.006881077038145101, 'mean_word_length': 3.9970104633781762, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 4, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5958561', 'n_tokens_mistral': 2296, 'n_tokens_neox': 2279, 'n_words': 577}