<commit_before># coding=utf-8
import os
import shutil

from django.db.models import ImageField
from django.core.files.images import ImageFile
from django.conf import settings
from PIL import Image

from hashing import hexHash

max_image_size = getattr(settings, 'MAX_IMAGE_SIZE', 1200)


def get_parameters(image_path):
    """
    Parses the image path to dictionary
    :param str image_path: image path
    :rtype dict
    """
    image_path = image_path
    image_directory = os.path.dirname(image_path)
    image_filename = os.path.basename(image_path)
    image_name = image_filename.split('.')[0]
    image_extension = image_filename.split('.')[-1]
    return {
        'directory': image_directory,
        'extension': image_extension,
        'name': image_name,
        'filename': image_filename,
        'path': image_path
    }


def fix_image_path(image_path):
    """
    Fixes the image path, transforming it to {md5hash}.{extension}

    :param str image_path: image path
    :rtype str
    """
    image = get_parameters(image_path)

    # early extension fix
    image['extension'] = image['extension'].lower().replace('jpeg', 'jpg')
    # test
    wrong_name = False
    try:                          # check ascii characters in filename
        image['name'].encode('ascii')
    except UnicodeEncodeError:
        wrong_name = True
    except UnicodeDecodeError:
        wrong_name = True

    if len(image['name']) != 32:  # check image name length
        wrong_name = True

    if wrong_name:
        image['name'] = hexHash(image['name'])

    return os.path.join(image['directory'], ''.join([image['name'], '.', image['extension']]))


def normalize_image_size(image_path, max_size):
    """
    Reduces image size to max_size
    :param image_path: path to image
    :param max_size: maximum size
    """
    try:
        image_object = Image.open(image_path)
    except IOError:
        raise IOError("No image found at %s" % image_path)
    image = get_parameters(image_path)
    if max(image_object.size) > max_size:
        resize_rate = max(image_object.size) / float(max_size)
        new_size = (int(image_object.size[0] / resize_rate), int(image_object.size[1] / resize_rate))
        image_object.thumbnail(new_size, Image.ANTIALIAS)
        new_path = os.path.join(image['directory'], ''.join([image['name'], '_thumbnail.jpg']))
        try:
            if image_object.mode != "RGB":
                image_object = image_object.convert("RGB")
            image_object.save(new_path, quality=70)
            shutil.copy(new_path, image['path'])
        except (IOError, KeyError, ValueError):
            try:
                os.remove(new_path)
            except IOError:
                pass
            raise


def save(image_field):
    image_path = image_field.path.encode("utf-8")

    try:
        image_exists = os.path.isfile(image_path)
    except (UnicodeEncodeError, UnicodeDecodeError):
        raise IOError('Incorrect locale settings')

    if not image_exists:
        return

    image_path = image_field.path
    new_image_path = fix_image_path(image_path)
    if new_image_path != image_path:
        image_content = ImageFile(open(image_path))
        try:
            image_field.save(os.path.basename(image_path), image_content, save=True)
        except IOError:
            pass    # TODO: process error
        image_path = new_image_path

    if not os.path.isfile(image_path):
        raise IOError('Failed to create %s' % image_path)
    try:
        normalize_image_size(image_path, max_image_size)
    except IOError:
        raise IOError('Failed to process %s' % image_path)


def upload_to(path):
    """
    Возвращает функцию генерации пути
    """

    def update_filename(instance, filename):
        f = fix_image_path(filename)<commit_msg>added unique hash<commit_after># coding=utf-8
import os
import shutil

from django.db.models import ImageField
from django.core.files.images import ImageFile
from django.conf import settings
from PIL import Image

from hashing import hexHash
import time

max_image_size = getattr(settings, 'MAX_IMAGE_SIZE', 1200)


def get_parameters(image_path):
    """
    Parses the image path to dictionary
    :param str image_path: image path
    :rtype dict
    """
    image_path = image_path
    image_directory = os.path.dirname(image_path)
    image_filename = os.path.basename(image_path)
    image_name = image_filename.split('.')[0]
    image_extension = image_filename.split('.')[-1]
    return {
        'directory': image_directory,
        'extension': image_extension,
        'name': image_name,
        'filename': image_filename,
        'path': image_path
    }


def fix_image_path(image_path):
    """
    Fixes the image path, transforming it to {md5hash}.{extension}

    :param str image_path: image path
    :rtype str
    """
    image = get_parameters(image_path)

    # early extension fix
    image['extension'] = image['extension'].lower().replace('jpeg', 'jpg')
    # test
    wrong_name = False
    try:                          # check ascii characters in filename
        image['name'].encode('ascii')
    except UnicodeError:
        wrong_name = True

    if len(image['name']) != 32:  # check image name length
        wrong_name = True

    if wrong_name:
        image['name'] = hexHash(image['name'] + str(time.time()))

    return os.path.join(image['directory'], ''.join([image['name'], '.', image['extension']]))


def normalize_image_size(image_path, max_size):
    """
    Reduces image size to max_size
    :param image_path: path to image
    :param max_size: maximum size
    """
    try:
        image_object = Image.open(image_path)
    except IOError:
        raise IOError("No image found at %s" % image_path)
    image = get_parameters(image_path)
    if max(image_object.size) > max_size:
        resize_rate = max(image_object.size) / float(max_size)
        new_size = (int(image_object.size[0] / resize_rate), int(image_object.size[1] / resize_rate))
        image_object.thumbnail(new_size, Image.ANTIALIAS)
        new_path = os.path.join(image['directory'], ''.join([image['name'], '_thumbnail.jpg']))
        try:
            if image_object.mode != "RGB":
                image_object = image_object.convert("RGB")
            image_object.save(new_path, quality=70)
            shutil.copy(new_path, image['path'])
        except (IOError, KeyError, ValueError):
            try:
                os.remove(new_path)
            except IOError:
                pass
            raise


def save(image_field):
    image_path = image_field.path.encode("utf-8")

    try:
        image_exists = os.path.isfile(image_path)
    except UnicodeError:
        raise IOError('Incorrect locale settings')

    if not image_exists:
        return

    image_path = image_field.path
    new_image_path = fix_image_path(image_path)
    if new_image_path != image_path:
        image_content = ImageFile(open(image_path))
        try:
            image_field.save(os.path.basename(image_path), image_content, save=True)
        except IOError:
            pass    # TODO: process error
        image_path = new_image_path

    if not os.path.isfile(image_path):
        raise IOError('Failed to create %s' % image_path)
    try:
        normalize_image_size(image_path, max_image_size)
    except IOError:
        raise IOError('Failed to process %s' % image_path)


def upload_to(path):
    """
    Возвращает функцию генерации пути
    """

    def update_filename(instance, filename):
        f = fix_image_path(filename)
{'fraction_non_alphanumeric': 0.10781208762206387, 'fraction_numerical': 0.00395882818685669, 'mean_word_length': 2.804718875502008, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5069256', 'n_tokens_mistral': 2349, 'n_tokens_neox': 2293, 'n_words': 527}