<commit_before># Node models to represent channel's tree
from __future__ import unicode_literals

from cachecontrol.caches.file_cache import FileCache
import hashlib
import os
from PIL import Image
from requests.exceptions import MissingSchema, HTTPError, ConnectionError, InvalidURL, InvalidSchema
import shutil
from subprocess import CalledProcessError
import tempfile
import youtube_dl
import zipfile

from le_utils.constants import languages
from le_utils.constants import file_formats, format_presets, exercises
from pressurecooker.encodings import get_base64_encoding, write_base64_to_file
from pressurecooker.images import create_image_from_pdf_page
from pressurecooker.images import create_image_from_epub
from pressurecooker.images import create_image_from_zip
from pressurecooker.videos import extract_thumbnail_from_video
from pressurecooker.images import create_waveform_image
from pressurecooker.images import create_tiled_image
from pressurecooker.images import ThumbnailGenerationError
from pressurecooker.subtitles import build_subtitle_converter_from_file
from pressurecooker.subtitles import LANGUAGE_CODE_UNKNOWN
from pressurecooker.subtitles import InvalidSubtitleFormatError, InvalidSubtitleLanguageError
from pressurecooker.videos import guess_video_preset_by_resolution, compress_video, VideoCompressionError
from pressurecooker.youtube import YouTubeResource

from .. import config
from ..exceptions import UnknownFileTypeError

# Cache for filenames
FILECACHE = FileCache(config.FILECACHE_DIRECTORY, forever=True)
HTTP_CAUGHT_EXCEPTIONS = (HTTPError, ConnectionError, InvalidURL, UnicodeDecodeError, UnicodeError, InvalidSchema, IOError, AssertionError)

# Lookup table for convertible file formats for a given preset
# used for converting avi/flv/etc. videos and srt subtitles
CONVERTIBLE_FORMATS = {p.id: p.convertible_formats for p in format_presets.PRESETLIST}


def extract_path_ext(path, default_ext=None):
    """
    Extract file extension (without dot) from `path` or return `default_ext` if
    path does not contain a valid extension.
    """
    _, dotext = os.path.splitext(path)
    if dotext:
        ext = dotext[1:]
        if len(ext) > 3 and '?' in ext:   # strip off any querystring if present
            ext = ext.split('?')[0]
    else:
        ext = None
    if not ext and default_ext:
        ext = default_ext
    if not ext:
        raise ValueError('No extension in path {} and default_ext is None'.format(path))
    return ext.lower()


def generate_key(action, path_or_id, settings=None, default=" (default)"):
    """ generate_key: generate key used for caching
        Args:
            action (str): how video is being processed (e.g. COMPRESSED or DOWNLOADED)
            path_or_id (str): path to video or youtube_id
            settings (dict): settings for compression or downloading passed in by user
            default (str): if settings are None, default to this extension (avoid overwriting keys)
        Returns: filename
    """
    settings = " {}".format(str(sorted(settings.items()))) if settings else default
    return "{}: {}{}".format(action.upper(), path_or_id, settings)


def download(path, default_ext=None):
    """
    Download `path` and save to storage based on file extension derived from `path`.
    :param path: An URL or a local path
    :param default_ext: fallback ext for file when path does not end with .ext
    :return: filename derived from hash of file contents {md5hash(file)}.ext
    :rtype: sting (path of the form `{md5hash(file at path)}.ext`
    """
    key = "DOWNLOAD:{}".format(path)
    if not config.UPDATE and FILECACHE.get(key):
        return FILECACHE.get(key).decode('utf-8')

    config.LOGGER.info("\tDownloading {}".format(path))

    # Write file to temporary file
    with tempfile.TemporaryFile() as tempf:
        hash = write_and_get_hash(path, tempf)
        tempf.seek(0)
        # Get extension of file or use `default_ext` if none found
        ext = extract_path_ext(path, default_ext=default_ext)
        filename = '{0}.{ext}'.format(hash.hexdigest(), ext=ext)
        copy_file_to_storage(filename, tempf)
        FILECACHE.set(key, bytes(filename, "utf-8"))
        return filename


def download_and_convert_video(path, default_ext=file_formats.MP4, ffmpeg_settings=None):
    """
    Auto-converting variant of download function that handles all video formats.
    """
    ffmpeg_settings = ffmpeg_settings or {}
    key = "DOWNLOAD:{}".format(path)
    if not config.UPDATE and FILECACHE.get(key):
        return FILECACHE.get(key).decode('utf-8')

    config.LOGGER.info("\tDownloading {}".format(path))

    # Get extension of convertible video file
    ext = extract_path_ext(path)
    converted_path = None

    # Convert video to temporary file
    with tempfile.NamedTemporaryFile(suffix=".{}".format(ext), delete=False) as tempf:
        # Write unsupported video to temporary file
        write_and_get_hash(path, tempf)
        tempf.seek(0)
        # Compress video into mp4 file
        path, _ext = os.path.splitext(tempf.name)
        converted_path = "{}.{}".format(path, file_formats.MP4)
        tempf.close()
        compress_video(tempf.name, converted_path, overwrite=True, **ffmpeg_settings)
        os.unlink(tempf.name)

    # Write converted file to another temporary file
    with tempfile.TemporaryFile() as tempf2:
        hash = write_and_get_hash(converted_path, tempf2)
        tempf2.seek(0)
        filename = '{0}.{ext}'.format(hash.hexdigest(), ext=file_formats.MP4)
        copy_file_to_storage(filename, tempf2)
        os.unlink(converted_path)
        FILECACHE.set(key, bytes(filename, "utf-8"))
        return filename


def write_and_get_hash(path, write_to_file, hash=None):
    """
    Download file at `path` and write its contents to the file `write_to_file`.
    Attempts a HTTP GET request for the path and if that fails, we'll try to read
    from `path` on the local filesystem.

    :param path: An URL or a local filepath
    :type path: str
    :param write_to_file: Desrination file to which we write the contents of path.
    :type write_to_file: file
    :param hash:  initial hasher state (used when writing multiple paths to the same file (.graphie)
    :type hash: hashlib hasher (optional)
    :return: updated hasher state
    :rtype: hashlib hasher (updated)
    """
    hash = hash or hashlib.md5()
    try:
        # Access path
        r = config.DOWNLOAD_SESSION.get(path, stream=True)
        r.raise_for_status()
        for chunk in r:
            write_to_file.write(chunk)
            hash.update(chunk)

    except (MissingSchema, InvalidSchema):
        # If path is a local file path, try to open the file (generate hash if none provided)
        with open(path, 'rb') as fobj:
            for chunk in iter(lambda: fobj.read(2097152), b""):
                write_to_file.write(chunk)
                hash.update(chunk)

    assert write_to_file.tell() > 0, "File failed to write (corrupted)."

    return hash


def copy_file_to_storage(filename, srcfile, delete_original=False):
    """
    Copy `srcfile` (file of path) to desrination storage path for `filename`,
    of the form `storage/f/i/filename.ext` when filename is `filename.ext`.
    :rtype: None
    """
    # Some files might have been closed, so only filepath will work
    if isinstance(srcfile, str):
        srcfile = open(srcfile, 'rb')

    # Write file to local storage
    with open(config.get_storage_path(filename), 'wb') as destf:
        if delete_original:
            shutil.move(srcfile.name, destf.name)
        else:
            shutil.copyfileobj(srcfile, destf)


def get_hash(filepath):
    file_hash = hashlib.md5()
    with open(filepath, 'rb') as fobj:
        for chunk in iter(lambda: fobj.read(2097152), b""):
            file_hash.update(chunk)
    return file_hash.hexdigest()




def compress_video_file(filename, ffmpeg_settings):
    """
    Calls the pressurecooker function `compress_video` to compress filename (source)
    stored in storage. Returns the filename of the compressed file.
    """
    ffmpeg_settings = ffmpeg_settings or {}
    key = generate_key("COMPRESSED", filename, settings=ffmpeg_settings, default=" (default compression)")

    if not config.UPDATE and FILECACHE.get(key):
        return FILECACHE.get(key).decode('utf-8')

    config.LOGGER.info("\t--- Compressing {}".format(filename))

    tempf = tempfile.NamedTemporaryFile(suffix=".{}".format(file_formats.MP4), delete=False)
    tempf.close() # Need to close so pressure cooker can write to file
    compress_video(config.get_storage_path(filename), tempf.name, overwrite=True, **ffmpeg_settings)
    compressedfilename = "{}.{}".format(get_hash(tempf.name), file_formats.MP4)

    copy_file_to_storage(compressedfilename, tempf.name)
    os.unlink(tempf.name)
    FILECACHE.set(key, bytes(compressedfilename, "utf-8"))
    return compressedfilename




def download_from_web(web_url, download_settings, file_format=file_formats.MP4, ext="", download_ext=""):
    """
    Download `web_url` using YoutubeDL using `download_settings` options.
    Required args: file_format, ext, and download_ext.
    """
    key = generate_key("DOWNLOADED", web_url, settings=download_settings)
    if not config.UPDATE and FILECACHE.get(key):
        return FILECACHE.get(key).decode('utf-8')

    # Get hash of web_url to act as temporary storage name
    url_hash = hashlib.md5()
    url_hash.update(web_url.encode('utf-8'))
    destination_path = os.path.join(tempfile.gettempdir(), "{}{ext}".format(url_hash.hexdigest(), ext=ext))
    download_settings["outtmpl"] = destination_path
    try:
        os.remove(destination_path)
    except Exception:
        pass

    # Download web_url (can be either a video or a video subtitles resource)
    if not config.USEPROXY:
        # Connect to YouTube directly
        with youtube_dl.YoutubeDL(download_settings) as ydl:
            ydl.download([web_url])
    else:
        # Connect to YouTube via an HTTP proxy
        yt_resource = YouTubeResource(web_url, useproxy=True, options=download_settings)
        result1 = yt_resource.get_resource_info()
        if result1 is None:
            raise youtube_dl.utils.DownloadError('Failed to get resource info')
        download_settings["writethumbnail"] = False      # overwrite default download behaviour
        download_settings["outtmpl"] = destination_path  # overwrite default download behaviour
        if file_format == file_formats.VTT:
            # We need to use the proxy when downloading subtitles
            result2 = yt_resource.download(options=download_settings, useproxy=True)
        else:
            # For video files we can skip the proxy for faster download speed
            result2 = yt_resource.download(options=download_settings)
        if result2 is None:
            raise youtube_dl.utils.DownloadError('Failed to download resource')

    destination_path += download_ext
    filename = "{}.{}".format(get_hash(destination_path), file_format)

    # Write file to local storage
    with open(destination_path, "rb") as dlf, open(config.get_storage_path(filename), 'wb') as destf:
        shutil.copyfileobj(dlf, destf)<commit_msg>robustify download logic -- make sure destination_path file exists<commit_after># Node models to represent channel's tree
from __future__ import unicode_literals

from cachecontrol.caches.file_cache import FileCache
import hashlib
import os
from PIL import Image
from requests.exceptions import MissingSchema, HTTPError, ConnectionError, InvalidURL, InvalidSchema
import shutil
from subprocess import CalledProcessError
import tempfile
import youtube_dl
import zipfile

from le_utils.constants import languages
from le_utils.constants import file_formats, format_presets, exercises
from pressurecooker.encodings import get_base64_encoding, write_base64_to_file
from pressurecooker.images import create_image_from_pdf_page
from pressurecooker.images import create_image_from_epub
from pressurecooker.images import create_image_from_zip
from pressurecooker.videos import extract_thumbnail_from_video
from pressurecooker.images import create_waveform_image
from pressurecooker.images import create_tiled_image
from pressurecooker.images import ThumbnailGenerationError
from pressurecooker.subtitles import build_subtitle_converter_from_file
from pressurecooker.subtitles import LANGUAGE_CODE_UNKNOWN
from pressurecooker.subtitles import InvalidSubtitleFormatError, InvalidSubtitleLanguageError
from pressurecooker.videos import guess_video_preset_by_resolution, compress_video, VideoCompressionError
from pressurecooker.youtube import YouTubeResource

from .. import config
from ..exceptions import UnknownFileTypeError

# Cache for filenames
FILECACHE = FileCache(config.FILECACHE_DIRECTORY, forever=True)
HTTP_CAUGHT_EXCEPTIONS = (HTTPError, ConnectionError, InvalidURL, UnicodeDecodeError, UnicodeError, InvalidSchema, IOError, AssertionError)

# Lookup table for convertible file formats for a given preset
# used for converting avi/flv/etc. videos and srt subtitles
CONVERTIBLE_FORMATS = {p.id: p.convertible_formats for p in format_presets.PRESETLIST}


def extract_path_ext(path, default_ext=None):
    """
    Extract file extension (without dot) from `path` or return `default_ext` if
    path does not contain a valid extension.
    """
    _, dotext = os.path.splitext(path)
    if dotext:
        ext = dotext[1:]
        if len(ext) > 3 and '?' in ext:   # strip off any querystring if present
            ext = ext.split('?')[0]
    else:
        ext = None
    if not ext and default_ext:
        ext = default_ext
    if not ext:
        raise ValueError('No extension in path {} and default_ext is None'.format(path))
    return ext.lower()


def generate_key(action, path_or_id, settings=None, default=" (default)"):
    """ generate_key: generate key used for caching
        Args:
            action (str): how video is being processed (e.g. COMPRESSED or DOWNLOADED)
            path_or_id (str): path to video or youtube_id
            settings (dict): settings for compression or downloading passed in by user
            default (str): if settings are None, default to this extension (avoid overwriting keys)
        Returns: filename
    """
    settings = " {}".format(str(sorted(settings.items()))) if settings else default
    return "{}: {}{}".format(action.upper(), path_or_id, settings)


def download(path, default_ext=None):
    """
    Download `path` and save to storage based on file extension derived from `path`.
    :param path: An URL or a local path
    :param default_ext: fallback ext for file when path does not end with .ext
    :return: filename derived from hash of file contents {md5hash(file)}.ext
    :rtype: sting (path of the form `{md5hash(file at path)}.ext`
    """
    key = "DOWNLOAD:{}".format(path)
    if not config.UPDATE and FILECACHE.get(key):
        return FILECACHE.get(key).decode('utf-8')

    config.LOGGER.info("\tDownloading {}".format(path))

    # Write file to temporary file
    with tempfile.TemporaryFile() as tempf:
        hash = write_and_get_hash(path, tempf)
        tempf.seek(0)
        # Get extension of file or use `default_ext` if none found
        ext = extract_path_ext(path, default_ext=default_ext)
        filename = '{0}.{ext}'.format(hash.hexdigest(), ext=ext)
        copy_file_to_storage(filename, tempf)
        FILECACHE.set(key, bytes(filename, "utf-8"))
        return filename


def download_and_convert_video(path, default_ext=file_formats.MP4, ffmpeg_settings=None):
    """
    Auto-converting variant of download function that handles all video formats.
    """
    ffmpeg_settings = ffmpeg_settings or {}
    key = "DOWNLOAD:{}".format(path)
    if not config.UPDATE and FILECACHE.get(key):
        return FILECACHE.get(key).decode('utf-8')

    config.LOGGER.info("\tDownloading {}".format(path))

    # Get extension of convertible video file
    ext = extract_path_ext(path)
    converted_path = None

    # Convert video to temporary file
    with tempfile.NamedTemporaryFile(suffix=".{}".format(ext), delete=False) as tempf:
        # Write unsupported video to temporary file
        write_and_get_hash(path, tempf)
        tempf.seek(0)
        # Compress video into mp4 file
        path, _ext = os.path.splitext(tempf.name)
        converted_path = "{}.{}".format(path, file_formats.MP4)
        tempf.close()
        compress_video(tempf.name, converted_path, overwrite=True, **ffmpeg_settings)
        os.unlink(tempf.name)

    # Write converted file to another temporary file
    with tempfile.TemporaryFile() as tempf2:
        hash = write_and_get_hash(converted_path, tempf2)
        tempf2.seek(0)
        filename = '{0}.{ext}'.format(hash.hexdigest(), ext=file_formats.MP4)
        copy_file_to_storage(filename, tempf2)
        os.unlink(converted_path)
        FILECACHE.set(key, bytes(filename, "utf-8"))
        return filename


def write_and_get_hash(path, write_to_file, hash=None):
    """
    Download file at `path` and write its contents to the file `write_to_file`.
    Attempts a HTTP GET request for the path and if that fails, we'll try to read
    from `path` on the local filesystem.

    :param path: An URL or a local filepath
    :type path: str
    :param write_to_file: Desrination file to which we write the contents of path.
    :type write_to_file: file
    :param hash:  initial hasher state (used when writing multiple paths to the same file (.graphie)
    :type hash: hashlib hasher (optional)
    :return: updated hasher state
    :rtype: hashlib hasher (updated)
    """
    hash = hash or hashlib.md5()
    try:
        # Access path
        r = config.DOWNLOAD_SESSION.get(path, stream=True)
        r.raise_for_status()
        for chunk in r:
            write_to_file.write(chunk)
            hash.update(chunk)

    except (MissingSchema, InvalidSchema):
        # If path is a local file path, try to open the file (generate hash if none provided)
        with open(path, 'rb') as fobj:
            for chunk in iter(lambda: fobj.read(2097152), b""):
                write_to_file.write(chunk)
                hash.update(chunk)

    assert write_to_file.tell() > 0, "File failed to write (corrupted)."

    return hash


def copy_file_to_storage(filename, srcfile, delete_original=False):
    """
    Copy `srcfile` (file of path) to desrination storage path for `filename`,
    of the form `storage/f/i/filename.ext` when filename is `filename.ext`.
    :rtype: None
    """
    # Some files might have been closed, so only filepath will work
    if isinstance(srcfile, str):
        srcfile = open(srcfile, 'rb')

    # Write file to local storage
    with open(config.get_storage_path(filename), 'wb') as destf:
        if delete_original:
            shutil.move(srcfile.name, destf.name)
        else:
            shutil.copyfileobj(srcfile, destf)


def get_hash(filepath):
    file_hash = hashlib.md5()
    with open(filepath, 'rb') as fobj:
        for chunk in iter(lambda: fobj.read(2097152), b""):
            file_hash.update(chunk)
    return file_hash.hexdigest()




def compress_video_file(filename, ffmpeg_settings):
    """
    Calls the pressurecooker function `compress_video` to compress filename (source)
    stored in storage. Returns the filename of the compressed file.
    """
    ffmpeg_settings = ffmpeg_settings or {}
    key = generate_key("COMPRESSED", filename, settings=ffmpeg_settings, default=" (default compression)")

    if not config.UPDATE and FILECACHE.get(key):
        return FILECACHE.get(key).decode('utf-8')

    config.LOGGER.info("\t--- Compressing {}".format(filename))

    tempf = tempfile.NamedTemporaryFile(suffix=".{}".format(file_formats.MP4), delete=False)
    tempf.close() # Need to close so pressure cooker can write to file
    compress_video(config.get_storage_path(filename), tempf.name, overwrite=True, **ffmpeg_settings)
    compressedfilename = "{}.{}".format(get_hash(tempf.name), file_formats.MP4)

    copy_file_to_storage(compressedfilename, tempf.name)
    os.unlink(tempf.name)
    FILECACHE.set(key, bytes(compressedfilename, "utf-8"))
    return compressedfilename




def download_from_web(web_url, download_settings, file_format=file_formats.MP4, ext="", download_ext=""):
    """
    Download `web_url` using YoutubeDL using `download_settings` options.
    Required args: file_format, ext, and download_ext.
    """
    key = generate_key("DOWNLOADED", web_url, settings=download_settings)
    if not config.UPDATE and FILECACHE.get(key):
        return FILECACHE.get(key).decode('utf-8')

    # Get hash of web_url to act as temporary storage name
    url_hash = hashlib.md5()
    url_hash.update(web_url.encode('utf-8'))
    tempfilename = "{}{ext}".format(url_hash.hexdigest(), ext=ext)
    destination_path = os.path.join(tempfile.gettempdir(), tempfilename)
    download_settings["outtmpl"] = destination_path
    if os.path.exists(destination_path):
        os.remove(destination_path)

    # Download web_url (can be either a video or a video subtitles resource)
    if not config.USEPROXY:
        # Connect to YouTube directly
        with youtube_dl.YoutubeDL(download_settings) as ydl:
            ydl.download([web_url])
            if not os.path.exists(destination_path):
                raise youtube_dl.utils.DownloadError('Failed to download resource ' + web_url)
    else:
        # Connect to YouTube via an HTTP proxy
        yt_resource = YouTubeResource(web_url, useproxy=True, options=download_settings)
        result1 = yt_resource.get_resource_info()
        if result1 is None:
            raise youtube_dl.utils.DownloadError('Failed to get resource info')
        download_settings["writethumbnail"] = False      # overwrite default download behaviour
        download_settings["outtmpl"] = destination_path  # overwrite default download behaviour
        if file_format == file_formats.VTT:
            # We need to use the proxy when downloading subtitles
            result2 = yt_resource.download(options=download_settings, useproxy=True)
        else:
            # For video files we can skip the proxy for faster download speed
            result2 = yt_resource.download(options=download_settings)
        if result2 is None or not os.path.exists(destination_path):
            raise youtube_dl.utils.DownloadError('Failed to download resource ' + web_url)

    destination_path += download_ext
    filename = "{}.{}".format(get_hash(destination_path), file_format)

    # Write file to local storage
    with open(destination_path, "rb") as dlf, open(config.get_storage_path(filename), 'wb') as destf:
        shutil.copyfileobj(dlf, destf)
{'fraction_non_alphanumeric': 0.09129269043118984, 'fraction_numerical': 0.004922860533602918, 'mean_word_length': 3.5972923822994542, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 71, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1753197', 'n_tokens_mistral': 6821, 'n_tokens_neox': 6642, 'n_words': 1947}