<commit_before>#!/usr/bin/env python3
# -*- coding: utf-8 -*-

from __future__ import unicode_literals

import configparser
import logging
import os
import shutil
from urllib.parse import urljoin
from uuid import uuid4

import mutagen.id3
# import shelve
import pkg_resources
import youtube_dl
from boltons.urlutils import find_all_links
from plumbum import local
from pydub import AudioSegment
from telegram import MessageEntity, InlineQueryResultCachedAudio, ChatAction, InlineKeyboardMarkup, InlineKeyboardButton
from telegram.contrib.botan import Botan
from telegram.ext import Updater, CommandHandler, MessageHandler, Filters, InlineQueryHandler, CallbackQueryHandler

# from transliterate import translit

logging.basicConfig(format='%(asctime)s - %(name)s - %(levelname)s - %(message)s', level=logging.DEBUG)
logger = logging.getLogger(__name__)
logger.setLevel(logging.DEBUG)

TG_BOT_TOKEN = os.environ['TG_BOT_TOKEN']
BOTAN_TOKEN = os.getenv('BOTAN_TOKEN', '')
STORE_CHAT_ID = os.environ['STORE_CHAT_ID']
SC_AUTH_TOKEN = os.environ['SC_AUTH_TOKEN']
NO_CLUTTER_CHAT_IDS = list(map(int, os.getenv('NO_CLUTTER_CHAT_IDS', '').split(',')))
DL_DIR = os.path.expanduser(os.getenv('DL_DIR', '~'))
USE_WEBHOOK = int(os.getenv('USE_WEBHOOK', '0'))
PORT = int(os.getenv('PORT', '5000'))
APP_URL = os.getenv('APP_URL', '')

MAX_TG_FILE_SIZE = 45000000
DL_EVENT_NAME = 'Download'
WAIT_TEXT = "Wait a bit.."
WAIT_TEXT_MD = "".join(["_", WAIT_TEXT, "_"])
ERROR_TEXT_MD = "_Sorry, something went wrong_"
texts = {}  # TODO shelve

scdl = local[os.path.join(os.getenv('BIN_PATH', ''), 'scdl')]
bcdl = local[os.path.join(os.getenv('BIN_PATH', ''), 'bandcamp-dl')]
bcdl_template = "%{artist} - %{track} - %{title} [%{album}]"

patterns = {
    "soundcloud": "soundcloud.com",
    "bandcamp": "bandcamp.com",
    "youtube": "youtube.com",
    "youtu.be": "youtu.be",
    "mixcloud": "mixcloud.com"
}

botan = Botan(BOTAN_TOKEN) if BOTAN_TOKEN else None
help_path = '/'.join(('messages', 'help.tg.md'))
help_message = pkg_resources.resource_string(__name__, help_path).decode("UTF-8")


def configure_scdl():
    config = configparser.ConfigParser()
    config['scdl'] = {
        'auth_token': SC_AUTH_TOKEN,
        'path': DL_DIR,
    }
    config_dir = os.path.join(os.path.expanduser('~'), '.config/scdl')
    config_path = os.path.join(config_dir, 'scdl.cfg')
    os.makedirs(config_dir, exist_ok=True)
    with open(config_path, 'w') as f:
        config.write(f)


def help_callback(bot, update):
    bot.send_message(chat_id=update.message.chat_id, text=help_message,
                     parse_mode='Markdown', disable_web_page_preview=True)


def download_callback(bot, update, args=None):
    global texts
    if update.inline_query:
        chat_id = STORE_CHAT_ID
        text = update.inline_query.query
    elif update.callback_query:
        chat_id = update.callback_query.message.chat_id
        if update.callback_query.data == "cancel":
            bot.delete_message(chat_id=chat_id, message_id=update.callback_query.message.message_id)
            return
        else:
            update.callback_query.answer(text=WAIT_TEXT)
            update.callback_query.edit_message_text(parse_mode='Markdown', text=WAIT_TEXT_MD)
            text = texts[update.callback_query.data]
    else:
        botan.track(update.message, event_name=DL_EVENT_NAME) if botan else None
        chat_id = update.message.chat_id
        if args:
            text = " ".join(args)
        else:
            text = update.message.text

    urls = find_all_links(text, default_scheme="http")
    str_urls = " ".join([url.to_text() for url in urls])  # TODO make it better

    if any((pattern in str_urls for pattern in patterns.values())):
        if args or update.inline_query or update.callback_query:
            reply_to_message_id = None
            wait_message_id = None
            if update.callback_query:
                wait_message_id = update.callback_query.message.message_id
            elif args or update.inline_query:
                if update.message and chat_id not in NO_CLUTTER_CHAT_IDS:
                    reply_to_message_id = update.message.message_id

                wait_message = bot.send_message(chat_id=chat_id, reply_to_message_id=reply_to_message_id,
                                                parse_mode='Markdown', text=WAIT_TEXT_MD)
                wait_message_id = wait_message.message_id

            download_dir = os.path.join(DL_DIR, str(uuid4()))
            shutil.rmtree(download_dir, ignore_errors=True)
            os.makedirs(download_dir)

            for url in urls:
                bot.send_chat_action(chat_id=chat_id, action=ChatAction.RECORD_AUDIO)
                download_audio(url, download_dir)

            file_list = []
            for d, dirs, files in os.walk(download_dir):
                for f in files:
                    file_list.append(os.path.join(d, f))
            file_list = sorted(file_list)

            sent_audio_ids = []
            for file in file_list:
                sent_audio_ids_file = send_audio(bot, chat_id, reply_to_message_id, file)
                sent_audio_ids.extend(sent_audio_ids_file)

            shutil.rmtree(download_dir, ignore_errors=True)
            bot.delete_message(chat_id=chat_id, message_id=wait_message_id)

            if not sent_audio_ids:
                bot.send_message(chat_id=chat_id, reply_to_message_id=reply_to_message_id,
                                 parse_mode='Markdown', text=ERROR_TEXT_MD)
                return
            else:
                if update.inline_query:
                    results = []
                    for audio_id in sent_audio_ids:
                        if audio_id:
                            results.append(
                                InlineQueryResultCachedAudio(
                                    id=str(uuid4()),
                                    audio_file_id=audio_id,
                                )
                            )
                    bot.answer_inline_query(update.inline_query.id, results)
        elif not args:
            reply_to_message_id = update.message.message_id
            texts[str(reply_to_message_id)] = update.message.text
            button_download = InlineKeyboardButton(text="Download", callback_data=str(reply_to_message_id))
            button_cancel = InlineKeyboardButton(text="Cancel", callback_data="cancel")
            inline_keyboard = InlineKeyboardMarkup([[button_download, button_cancel]])
            bot.send_message(chat_id=chat_id, reply_to_message_id=reply_to_message_id,
                             reply_markup=inline_keyboard, text="Wanna download?")


# @run_async
def download_audio(url, download_dir):
    url_parts_len = len([part for part in url.path_parts if part])
    if patterns["soundcloud"] in url.host:
        if 2 <= url_parts_len <= 3:
            scdl(
                "-l", url.to_text(full_quote=True),  # URL of track/playlist/user<commit_msg>urls_store<commit_after>#!/usr/bin/env python3
# -*- coding: utf-8 -*-

from __future__ import unicode_literals

import configparser
import logging
import os
import shutil
from urllib.parse import urljoin
from uuid import uuid4

import mutagen.id3
# import shelve
import pkg_resources
import youtube_dl
from boltons.urlutils import find_all_links
from plumbum import local
from pydub import AudioSegment
from telegram import MessageEntity, InlineQueryResultCachedAudio, ChatAction, InlineKeyboardMarkup, InlineKeyboardButton
from telegram.contrib.botan import Botan
from telegram.ext import Updater, CommandHandler, MessageHandler, Filters, InlineQueryHandler, CallbackQueryHandler

# from transliterate import translit

logging.basicConfig(format='%(asctime)s - %(name)s - %(levelname)s - %(message)s', level=logging.DEBUG)
logger = logging.getLogger(__name__)
logger.setLevel(logging.DEBUG)

TG_BOT_TOKEN = os.environ['TG_BOT_TOKEN']
BOTAN_TOKEN = os.getenv('BOTAN_TOKEN', '')
STORE_CHAT_ID = os.environ['STORE_CHAT_ID']
SC_AUTH_TOKEN = os.environ['SC_AUTH_TOKEN']
NO_CLUTTER_CHAT_IDS = list(map(int, os.getenv('NO_CLUTTER_CHAT_IDS', '').split(',')))
DL_DIR = os.path.expanduser(os.getenv('DL_DIR', '~'))
USE_WEBHOOK = int(os.getenv('USE_WEBHOOK', '0'))
PORT = int(os.getenv('PORT', '5000'))
APP_URL = os.getenv('APP_URL', '')

MAX_TG_FILE_SIZE = 45000000
DL_EVENT_NAME = 'Download'
WAIT_TEXT = "Wait a bit.."
WAIT_TEXT_MD = "".join(["_", WAIT_TEXT, "_"])
ERROR_TEXT_MD = "_Sorry, something went wrong_"
urls_store = {}  # TODO shelve

scdl = local[os.path.join(os.getenv('BIN_PATH', ''), 'scdl')]
bcdl = local[os.path.join(os.getenv('BIN_PATH', ''), 'bandcamp-dl')]
bcdl_template = "%{artist} - %{track} - %{title} [%{album}]"

patterns = {
    "soundcloud": "soundcloud.com",
    "bandcamp": "bandcamp.com",
    "youtube": "youtube.com",
    "youtu.be": "youtu.be",
    "mixcloud": "mixcloud.com"
}

botan = Botan(BOTAN_TOKEN) if BOTAN_TOKEN else None
help_path = '/'.join(('messages', 'help.tg.md'))
help_message = pkg_resources.resource_string(__name__, help_path).decode("UTF-8")


def configure_scdl():
    config = configparser.ConfigParser()
    config['scdl'] = {
        'auth_token': SC_AUTH_TOKEN,
        'path': DL_DIR,
    }
    config_dir = os.path.join(os.path.expanduser('~'), '.config/scdl')
    config_path = os.path.join(config_dir, 'scdl.cfg')
    os.makedirs(config_dir, exist_ok=True)
    with open(config_path, 'w') as f:
        config.write(f)


def help_callback(bot, update):
    bot.send_message(chat_id=update.message.chat_id, text=help_message,
                     parse_mode='Markdown', disable_web_page_preview=True)


def download_callback(bot, update, args=None):
    global urls_store
    urls = None
    text = None
    if update.inline_query:
        chat_id = STORE_CHAT_ID
        text = update.inline_query.query
    elif update.callback_query:
        chat_id = update.callback_query.message.chat_id
        if update.callback_query.data == "cancel":
            bot.delete_message(chat_id=chat_id, message_id=update.callback_query.message.message_id)
            return
        else:
            update.callback_query.answer(text=WAIT_TEXT)
            update.callback_query.edit_message_text(parse_mode='Markdown', text=WAIT_TEXT_MD)
            urls = urls_store[update.callback_query.data]
            urls_store.pop(update.callback_query.data)
    else:
        botan.track(update.message, event_name=DL_EVENT_NAME) if botan else None
        chat_id = update.message.chat_id
        if args:
            text = " ".join(args)
        else:
            text = update.message.text
    test = True
    if not urls:
        urls = find_all_links(text, default_scheme="http")
        str_urls = " ".join([url.to_text() for url in urls])  # TODO make it better
        test = any((pattern in str_urls for pattern in patterns.values()))

    if test:
        if args or update.inline_query or update.callback_query:
            reply_to_message_id = None
            wait_message_id = None
            if update.callback_query:
                wait_message_id = update.callback_query.message.message_id
            elif args or update.inline_query:
                if update.message and chat_id not in NO_CLUTTER_CHAT_IDS:
                    reply_to_message_id = update.message.message_id

                wait_message = bot.send_message(chat_id=chat_id, reply_to_message_id=reply_to_message_id,
                                                parse_mode='Markdown', text=WAIT_TEXT_MD)
                wait_message_id = wait_message.message_id

            download_dir = os.path.join(DL_DIR, str(uuid4()))
            shutil.rmtree(download_dir, ignore_errors=True)
            os.makedirs(download_dir)

            for url in urls:
                bot.send_chat_action(chat_id=chat_id, action=ChatAction.RECORD_AUDIO)
                download_audio(url, download_dir)

            file_list = []
            for d, dirs, files in os.walk(download_dir):
                for f in files:
                    file_list.append(os.path.join(d, f))
            file_list = sorted(file_list)

            sent_audio_ids = []
            for file in file_list:
                sent_audio_ids_file = send_audio(bot, chat_id, reply_to_message_id, file)
                sent_audio_ids.extend(sent_audio_ids_file)

            shutil.rmtree(download_dir, ignore_errors=True)
            bot.delete_message(chat_id=chat_id, message_id=wait_message_id)

            if not sent_audio_ids:
                bot.send_message(chat_id=chat_id, reply_to_message_id=reply_to_message_id,
                                 parse_mode='Markdown', text=ERROR_TEXT_MD)
                return
            else:
                if update.inline_query:
                    results = []
                    for audio_id in sent_audio_ids:
                        if audio_id:
                            results.append(
                                InlineQueryResultCachedAudio(
                                    id=str(uuid4()),
                                    audio_file_id=audio_id,
                                )
                            )
                    bot.answer_inline_query(update.inline_query.id, results)
        elif not args:
            reply_to_message_id = update.message.message_id
            urls_store[str(reply_to_message_id)] = urls
            button_download = InlineKeyboardButton(text="Download", callback_data=str(reply_to_message_id))
            button_cancel = InlineKeyboardButton(text="Cancel", callback_data="cancel")
            inline_keyboard = InlineKeyboardMarkup([[button_download, button_cancel]])
            bot.send_message(chat_id=chat_id, reply_to_message_id=reply_to_message_id,
                             reply_markup=inline_keyboard, text="Wanna download?")


# @run_async
def download_audio(url, download_dir):
    url_parts_len = len([part for part in url.path_parts if part])
    if patterns["soundcloud"] in url.host:
        if 2 <= url_parts_len <= 3:
            scdl(
                "-l", url.to_text(full_quote=True),  # URL of track/playlist/user
{'fraction_non_alphanumeric': 0.10381161162576905, 'fraction_numerical': 0.003111519694505339, 'mean_word_length': 2.6742010911925176, 'pattern_counts': {'":': 12, '<': 7, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 40, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '70626', 'n_tokens_mistral': 4517, 'n_tokens_neox': 4430, 'n_words': 803}