<commit_before>import time
import shutil
import os
from PIL import Image
import logging


def mkdir(newdir):
    """works the way a good mkdir should :)
        - already exists, silently complete
        - regular file in the way, raise an exception
        - parent directory(ies) does not exist, make them as well
    """
    if os.path.isdir(newdir):
        return
    if os.path.isfile(newdir):
        raise OSError("a file with the same name as the desired "
                      "dir, '%s', already exists." % newdir)
    head, tail = os.path.split(newdir)
    if head and not os.path.isdir(head):
        mkdir(head)
    if tail:
        os.mkdir(newdir)


_RESIZES = {}
_TIMESTAMPS = {}


def scale_and_crop(path, requested_size, row, col, zoom, image):
    im = Image.open(path)
    x, y = [float(v) for v in im.size]
    xr, yr = [float(v) for v in requested_size]
    r = min(xr / x, yr / y)

    box = (256 * row, 256 * col, 256 * (row + 1), 256 * (col + 1))

    w, h = int(round(x * r)), int(round(y * r))
    _cache_key = '%s-%s-%s' % (image, zoom, w)
    pathname, extension = os.path.splitext(path)

    _resized_file = path.replace(
        extension,
        '-%s-%s%s' % (zoom, w, extension)
    )

    already = _RESIZES.get(_cache_key)
    if already:
        im = already
    else:
        if os.path.isfile(_resized_file):
            print "REUSING", _resized_file
            logging.debug('REUSING %s' % _resized_file)
            im = Image.open(_resized_file)
            #print "Assert?", (im.size, (w,h))
        else:
            print "Need to resize..."
            im = im.resize((w, h),
                           resample=Image.ANTIALIAS)
            logging.debug("SAVE RESIZED TO %s" % _resized_file)
            #print "SAVE RESIZED TO", _resized_file
            im.save(_resized_file)

    _RESIZES[_cache_key] = im
    _TIMESTAMPS[_cache_key] = time.time()

    # to avoid memory bloat of `_RESIZES` getting too large, instead use the
    # `_TIMESTAMPS` dict to keep track of which Image instances are getting old
    _now = time.time()
    for key, timestamp in _TIMESTAMPS.items():
        age = _now - timestamp
        if age > 10:
            del _TIMESTAMPS[key]
            del _RESIZES[key]

    # convert (width, height, x, y) into PIL crop box
    return im.crop(box)


def make_thumbnail(*args, **kwargs):  # wrapper on _make_thumbnail()
    t0 = time.time()
    result = _make_thumbnail(*args, **kwargs)
    t1 = time.time()
    #logging.warning("%s seconds to make thumbnail for %s" % (t1-t0, args[0]))
    print "%s seconds to make thumbnail for %s" % (t1-t0, args[0])
    return result


def _make_thumbnail(image, width, extension, static_path):
    root = os.path.join(
        static_path,
        'uploads'
    )
    save_root = os.path.join(
        static_path,
        'thumbnails'
    )
    if not os.path.isdir(save_root):
        os.mkdir(save_root)

    path = os.path.join(root, image)
    for i in ('.png', '.jpg'):
        path = os.path.join(root, image + i)
        if os.path.isfile(path):
            break
    else:
        raise IOError(image)

    save_filepath = save_root
    save_filepath = os.path.join(save_filepath, image)
    if not os.path.isdir(save_filepath):
        mkdir(save_filepath)<commit_msg>desperate fix<commit_after>import time
import shutil
import os
from PIL import Image
import logging


def mkdir(newdir):
    """works the way a good mkdir should :)
        - already exists, silently complete
        - regular file in the way, raise an exception
        - parent directory(ies) does not exist, make them as well
    """
    if os.path.isdir(newdir):
        return
    if os.path.isfile(newdir):
        raise OSError("a file with the same name as the desired "
                      "dir, '%s', already exists." % newdir)
    head, tail = os.path.split(newdir)
    if head and not os.path.isdir(head):
        mkdir(head)
    if tail:
        os.mkdir(newdir)


_RESIZES = {}
_TIMESTAMPS = {}


def scale_and_crop(path, requested_size, row, col, zoom, image):
    im = Image.open(path)
    x, y = [float(v) for v in im.size]
    xr, yr = [float(v) for v in requested_size]
    r = min(xr / x, yr / y)

    box = (256 * row, 256 * col, 256 * (row + 1), 256 * (col + 1))

    w, h = int(round(x * r)), int(round(y * r))
    _cache_key = '%s-%s-%s' % (image, zoom, w)
    pathname, extension = os.path.splitext(path)

    _resized_file = path.replace(
        extension,
        '-%s-%s%s' % (zoom, w, extension)
    )

    already = _RESIZES.get(_cache_key)
    if already:
        im = already
    else:
        if os.path.isfile(_resized_file):
            print "REUSING", _resized_file
            logging.debug('REUSING %s' % _resized_file)
            im = Image.open(_resized_file)
            #print "Assert?", (im.size, (w,h))
        else:
            print "Need to resize..."
            im = im.resize((w, h),
                           resample=Image.ANTIALIAS)
            logging.debug("SAVE RESIZED TO %s" % _resized_file)
            #print "SAVE RESIZED TO", _resized_file
            im.save(_resized_file)

    _RESIZES[_cache_key] = im
    _TIMESTAMPS[_cache_key] = time.time()

    # to avoid memory bloat of `_RESIZES` getting too large, instead use the
    # `_TIMESTAMPS` dict to keep track of which Image instances are getting old
    _now = time.time()
    for key, timestamp in _TIMESTAMPS.items():
        age = _now - timestamp
        if age > 10:
            del _TIMESTAMPS[key]
            del _RESIZES[key]

    # convert (width, height, x, y) into PIL crop box
    return im.crop(box)


def make_thumbnail(*args, **kwargs):  # wrapper on _make_thumbnail()
    t0 = time.time()
    result = _make_thumbnail(*args, **kwargs)
    t1 = time.time()
    #logging.warning("%s seconds to make thumbnail for %s" % (t1-t0, args[0]))
    print "%s seconds to make thumbnail for %s" % (t1-t0, args[0])
    return result


def _make_thumbnail(image, width, extension, static_path,
                    raise_error_if_not_found=False):
    root = os.path.join(
        static_path,
        'uploads'
    )
    save_root = os.path.join(
        static_path,
        'thumbnails'
    )
    if not os.path.isdir(save_root):
        os.mkdir(save_root)

    path = os.path.join(root, image)
    for i in ('.png', '.jpg'):
        path = os.path.join(root, image + i)
        if os.path.isfile(path):
            break
    else:
        if raise_error_if_not_found:
            raise IOError(image)
        else:
            return

    save_filepath = save_root
    save_filepath = os.path.join(save_filepath, image)
    if not os.path.isdir(save_filepath):
        mkdir(save_filepath)
{'fraction_non_alphanumeric': 0.11974977658623771, 'fraction_numerical': 0.0071492403932082215, 'mean_word_length': 2.20831342570473, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2525996', 'n_tokens_mistral': 2357, 'n_tokens_neox': 2263, 'n_words': 627}