<commit_before>import copy
import os
import random
import shutil
import tempfile
import zipfile

from contentcuration.api import (write_file_to_storage,
                                 write_raw_content_to_storage)
from contentcuration.models import (File, generate_file_on_disk_name,
                                    generate_object_storage_name)
from django.core.files import File as DjFile
from django.core.files.storage import default_storage
from le_utils.constants import content_kinds, file_formats, format_presets
from pressurecooker.images import (create_image_from_pdf_page,
                                   create_tiled_image, create_waveform_image)
from pressurecooker.videos import compress_video, extract_thumbnail_from_video


def create_file_from_contents(contents, ext=None, node=None, preset_id=None, uploaded_by=None):
    checksum, _, path = write_raw_content_to_storage(contents, ext=ext)
    with default_storage.open(path, 'rb') as new_file:
        return File.objects.create(
            file_on_disk=DjFile(new_file),
            file_format_id=ext,
            file_size=default_storage.size(path),
            checksum=checksum,
            preset_id=preset_id,
            contentnode=node,
            uploaded_by=uploaded_by
        )


def get_file_diff(files):
    """Given a list of filenames as strings, find the filenames that aren't in our
    storage, and return.

    """
    storage = default_storage

    # Try to be storage system agnostic, in case we're using either the Object Storage,
    # or FileSystemStorage
    ret = []
    for f in files:
        filepath = generate_object_storage_name(os.path.splitext(f)[0], f)
        if not storage.exists(filepath) or storage.size(filepath) == 0:
            ret.append(f)

    return ret


def duplicate_file(file_object, node=None, assessment_item=None, preset_id=None, save=True):
    if not file_object:
        return None
    file_copy = copy.copy(file_object)
    file_copy.id = None
    file_copy.contentnode = node
    file_copy.assessment_item = assessment_item
    file_copy.preset_id = preset_id or file_object.preset_id
    if save:
        file_copy.save()
    return file_copy


def extract_thumbnail_wrapper(file_object, node=None, preset_id=None):
    ext = file_formats.PNG
    with tempfile.NamedTemporaryFile(suffix=".{}".format(ext)) as tempf, tempfile.NamedTemporaryFile(suffix=".{}".format(file_object.file_format.extension)) as localtempf:
        shutil.copyfileobj(file_object.file_on_disk, localtempf)
        localtempf.flush()
        tempf.close()<commit_msg>first pass optimization of get_file_diff<commit_after>import copy
import os
import random
import requests
import shutil
import tempfile
import zipfile
from multiprocessing.dummy import Pool

from contentcuration.api import (write_file_to_storage,
                                 write_raw_content_to_storage)
from contentcuration.models import (File, generate_file_on_disk_name,
                                    generate_object_storage_name)
from django.conf import settings
from django.core.files import File as DjFile
from django.core.files.storage import default_storage
from le_utils.constants import content_kinds, file_formats, format_presets
from pressurecooker.images import (create_image_from_pdf_page,
                                   create_tiled_image, create_waveform_image)
from pressurecooker.videos import compress_video, extract_thumbnail_from_video


def create_file_from_contents(contents, ext=None, node=None, preset_id=None, uploaded_by=None):
    checksum, _, path = write_raw_content_to_storage(contents, ext=ext)
    with default_storage.open(path, 'rb') as new_file:
        return File.objects.create(
            file_on_disk=DjFile(new_file),
            file_format_id=ext,
            file_size=default_storage.size(path),
            checksum=checksum,
            preset_id=preset_id,
            contentnode=node,
            uploaded_by=uploaded_by
        )


def get_file_diff(files):
    """Given a list of filenames as strings, find the filenames that aren't in our
    storage, and return.

    """
    storage = default_storage

    # We use a thread pool in here, making direct HEAD requests to the storage URL
    # to see if the objects exist.
    # The threaded method is found to be the fastest -- see
    # https://gist.github.com/aronasorman/57b8c01e5ed2b7cbf876e7734b7b9f38
    # for benchmarking details.
    ret = []

    session = requests.Session()
    def check_file_url(f):
        filepath = generate_object_storage_name(os.path.splitext(f)[0], f)
        url = "/".join([settings.AWS_S3_ENDPOINT_URL, settings.AWS_S3_BUCKET_NAME, filepath])
        resp = session.head(url)
        if resp.status_code != 200:
            ret.append(f)

    # use a pool of 3 threads to make our queries
    pool = Pool(3)
    pool.map(check_file_url, files)

    return ret


def duplicate_file(file_object, node=None, assessment_item=None, preset_id=None, save=True):
    if not file_object:
        return None
    file_copy = copy.copy(file_object)
    file_copy.id = None
    file_copy.contentnode = node
    file_copy.assessment_item = assessment_item
    file_copy.preset_id = preset_id or file_object.preset_id
    if save:
        file_copy.save()
    return file_copy


def extract_thumbnail_wrapper(file_object, node=None, preset_id=None):
    ext = file_formats.PNG
    with tempfile.NamedTemporaryFile(suffix=".{}".format(ext)) as tempf, tempfile.NamedTemporaryFile(suffix=".{}".format(file_object.file_format.extension)) as localtempf:
        shutil.copyfileobj(file_object.file_on_disk, localtempf)
        localtempf.flush()
        tempf.close()
{'fraction_non_alphanumeric': 0.07588266291937468, 'fraction_numerical': 0.005093975057087651, 'mean_word_length': 3.320182094081942, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3862021', 'n_tokens_mistral': 1778, 'n_tokens_neox': 1727, 'n_words': 395}