<commit_before>import logging
import os

import pkg_resources
import requests

try:
    import youtube_dl
    youtube_dl_bin_name = 'youtube-dl'
except:
    import youtube_dlc as youtube_dl
    youtube_dl_bin_name = 'youtube-dlc'

from boltons.urlutils import URL
from plumbum import local, ProcessExecutionError, ProcessTimedOut

from scdlbot.exceptions import *

# from requests.exceptions import Timeout, RequestException, SSLError

bin_path = os.getenv('BIN_PATH', '')
scdl_bin = local[os.path.join(bin_path, 'scdl')]
bandcamp_dl_bin = local[os.path.join(bin_path, 'bandcamp-dl')]
youtube_dl_bin = local[os.path.join(bin_path, youtube_dl_bin_name)]

BOTAN_TRACK_URL = 'https://api.botan.io/track'

logger = logging.getLogger(__name__)


def get_response_text(file_name):
    # https://stackoverflow.com/a/20885799/2490759
    path = '/'.join(('texts', file_name))
    return pkg_resources.resource_string(__name__, path).decode("UTF-8")


def get_direct_urls(url, cookies_file=None, cookies_download_file=None, source_ip=None, proxy=None):
    logger.debug("Entered get_direct_urls")
    youtube_dl_args = []

    # https://github.com/ytdl-org/youtube-dl#how-do-i-pass-cookies-to-youtube-dl
    if cookies_file:
        if "http" in cookies_file:
            try:
                r = requests.get(cookies_file, allow_redirects=True, timeout=5)
                open(cookies_download_file, 'wb').write(r.content)
                youtube_dl_args.extend(["--cookies", cookies_download_file])
            except:
                pass
        else:
            youtube_dl_args.extend(["--cookies", cookies_file])

    if source_ip:
        youtube_dl_args.extend(["--source-address", source_ip])

    if proxy:
        youtube_dl_args.extend(["--proxy", proxy])

    youtube_dl_args.extend(["--get-url", url])
    try:
        ret_code, std_out, std_err = youtube_dl_bin[youtube_dl_args].run(timeout=10)
    except ProcessTimedOut as exc:
        raise URLTimeoutError
    except ProcessExecutionError as exc:
        # TODO: look at case: one page has multiple videos, some available, some not
        if "returning it as such" in exc.stderr:
            raise URLDirectError
        if "proxy server" in exc.stderr:
            raise URLCountryError
        raise exc
    if "yt_live_broadcast" in std_out:
        raise URLLiveError
    return std_out


def get_italic(text):
    return "_{}_".format(text)


def youtube_dl_func(url, ydl_opts, queue=None):
    ydl = youtube_dl.YoutubeDL(ydl_opts)<commit_msg>youtube-dl get-url increase timeout<commit_after>import logging
import os

import pkg_resources
import requests

try:
    import youtube_dl
    youtube_dl_bin_name = 'youtube-dl'
except:
    import youtube_dlc as youtube_dl
    youtube_dl_bin_name = 'youtube-dlc'

from boltons.urlutils import URL
from plumbum import local, ProcessExecutionError, ProcessTimedOut

from scdlbot.exceptions import *

# from requests.exceptions import Timeout, RequestException, SSLError

bin_path = os.getenv('BIN_PATH', '')
scdl_bin = local[os.path.join(bin_path, 'scdl')]
bandcamp_dl_bin = local[os.path.join(bin_path, 'bandcamp-dl')]
youtube_dl_bin = local[os.path.join(bin_path, youtube_dl_bin_name)]

BOTAN_TRACK_URL = 'https://api.botan.io/track'

logger = logging.getLogger(__name__)


def get_response_text(file_name):
    # https://stackoverflow.com/a/20885799/2490759
    path = '/'.join(('texts', file_name))
    return pkg_resources.resource_string(__name__, path).decode("UTF-8")


def get_direct_urls(url, cookies_file=None, cookies_download_file=None, source_ip=None, proxy=None):
    logger.debug("Entered get_direct_urls")
    youtube_dl_args = []

    # https://github.com/ytdl-org/youtube-dl#how-do-i-pass-cookies-to-youtube-dl
    if cookies_file:
        if "http" in cookies_file:
            try:
                r = requests.get(cookies_file, allow_redirects=True, timeout=5)
                open(cookies_download_file, 'wb').write(r.content)
                youtube_dl_args.extend(["--cookies", cookies_download_file])
            except:
                pass
        else:
            youtube_dl_args.extend(["--cookies", cookies_file])

    if source_ip:
        youtube_dl_args.extend(["--source-address", source_ip])

    if proxy:
        youtube_dl_args.extend(["--proxy", proxy])

    youtube_dl_args.extend(["--get-url", url])
    try:
        ret_code, std_out, std_err = youtube_dl_bin[youtube_dl_args].run(timeout=60)
    except ProcessTimedOut as exc:
        raise URLTimeoutError
    except ProcessExecutionError as exc:
        # TODO: look at case: one page has multiple videos, some available, some not
        if "returning it as such" in exc.stderr:
            raise URLDirectError
        if "proxy server" in exc.stderr:
            raise URLCountryError
        raise exc
    if "yt_live_broadcast" in std_out:
        raise URLLiveError
    return std_out


def get_italic(text):
    return "_{}_".format(text)


def youtube_dl_func(url, ydl_opts, queue=None):
    ydl = youtube_dl.YoutubeDL(ydl_opts)
{'fraction_non_alphanumeric': 0.10670368901229671, 'fraction_numerical': 0.00753669178897263, 'mean_word_length': 3.385217391304348, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 6, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4499871', 'n_tokens_mistral': 1754, 'n_tokens_neox': 1632, 'n_words': 336}