<commit_before>import hashlib
import json
import os
import pickle

from datetime import date, datetime

import shutil
from django.conf import settings
from django.utils import timezone

BLOCKSIZE = 65536

def create_dated(filename):
    """
    Based on the filename will create a full path filename including the date and time in '%Y%m%d_%H%M' format.
    The path to the filename will be set in the TEST_OUTPUT_PATH settings variable.

    :param filename: base filename. my_excel_data.xlsx for example
    :return: string, full path to file with date and time in the TEST_OUTPUT_PATH folder
    """
    if getattr(settings, 'TEST_OUTPUT_PATH', None) is None:
        msg = 'You need a the variable TEST_OUTPUT_PATH in settings. It should point to a folder' \
              'for temporary data to be written and reviewed.'
        raise ValueError(msg)
    if not os.path.exists(settings.TEST_OUTPUT_PATH):
        os.makedirs(settings.TEST_OUTPUT_PATH)
    return add_date(os.path.join(settings.TEST_OUTPUT_PATH, filename))

def hash_file(filename, algorithm='sha1', block_size=BLOCKSIZE):
    try:
        hasher = getattr(hashlib, algorithm)()
    except AttributeError:
        raise ValueError('{} is not a valid hashing algorithm'.format(algorithm))

    with open(filename, 'rb') as afile:
        buf = afile.read(block_size)
        while len(buf) > 0:
            hasher.update(buf)
            buf = afile.read(block_size)
    return hasher.hexdigest()


def parametrized(dec):
    """
    Need to study this code.
    Got it from http://stackoverflow.com/questions/5929107/python-decorators-with-parameters
    :param dec:
    :return:
    """

    def layer(*args, **kwargs):
        def repl(f):
            return dec(f, *args, **kwargs)
        return repl
    return layer


@parametrized
def temporary_file(func, extension, delete_on_exit=True):
    """
    This method decorator creates a filename with date using the provided extension and delete the file after the method
    has been executed.

    The settings.TEST_OUTPUT_PATH must be configured in your settings file.

    .. code-block:: python

        @temporary_file('json')
        def test_temporary_file_decorator(self):
            filename = self.test_temporary_file_decorator.filename
            ... write to the file ...


    :param func: function to decorate
    :param extension: extension of the filename without the dot
    :param delete_on_exit: If True the filename will be deleted.
    :return: the function
    """
    filename = create_dated('{}.{}'.format(func.__name__, extension))

    def function_t_return(*args):
        results = func(*args)
        if os.path.exists(filename) and delete_on_exit:
            os.remove(filename)
        return results

    function_t_return.filename = filename
    return function_t_return


def json_serial(obj):
    """JSON serializer for objects not serializable by default json code
    taken from: https://stackoverflow.com/questions/11875770/how-to-overcome-datetime-datetime-not-json-serializable
    """

    if isinstance(obj, (datetime, date)):
        serial = obj.isoformat()
        return serial
    raise TypeError("Type %s not serializable" % type(obj))


def serialize_data(data, output_file=None, format='json', **kwargs):
    """
    Quick function to serialize a data to file. The data keys will be saved in an alphabetical order
    for consistency purposes.
    If no output_file is supplied the function will created a dated file in the settings.TEST_OUTPUT_PATH folder.
    if the output_file is a folder the dated file will be created on the supplied folder with the serialized date.
    if the output_file is a file the data will be serialized to thar file

    :param data: Dictionary or list to serialize
    :param format: Format to serialize to. Currently json is the only one supported
    :param output_file: File to output the data to
    :param kwargs:
    """
    assert format in ['json', 'pickle'], 'Unsupported format {}'.format(format)
    if output_file is None:
        filename = create_dated('{}.{}'.format('serialize_data_q', format))
    elif os.path.isdir(output_file):
        filename = os.path.join(output_file, '{}.{}'.format('serialize_data_f', format))
    else:<commit_msg>Added base_filename keyword argument to serialze_data<commit_after>import hashlib
import json
import os
import pickle

from datetime import date, datetime

import shutil
from django.conf import settings
from django.utils import timezone

BLOCKSIZE = 65536

def create_dated(filename):
    """
    Based on the filename will create a full path filename including the date and time in '%Y%m%d_%H%M' format.
    The path to the filename will be set in the TEST_OUTPUT_PATH settings variable.

    :param filename: base filename. my_excel_data.xlsx for example
    :return: string, full path to file with date and time in the TEST_OUTPUT_PATH folder
    """
    if getattr(settings, 'TEST_OUTPUT_PATH', None) is None:
        msg = 'You need a the variable TEST_OUTPUT_PATH in settings. It should point to a folder' \
              'for temporary data to be written and reviewed.'
        raise ValueError(msg)
    if not os.path.exists(settings.TEST_OUTPUT_PATH):
        os.makedirs(settings.TEST_OUTPUT_PATH)
    return add_date(os.path.join(settings.TEST_OUTPUT_PATH, filename))

def hash_file(filename, algorithm='sha1', block_size=BLOCKSIZE):
    try:
        hasher = getattr(hashlib, algorithm)()
    except AttributeError:
        raise ValueError('{} is not a valid hashing algorithm'.format(algorithm))

    with open(filename, 'rb') as afile:
        buf = afile.read(block_size)
        while len(buf) > 0:
            hasher.update(buf)
            buf = afile.read(block_size)
    return hasher.hexdigest()


def parametrized(dec):
    """
    Need to study this code.
    Got it from http://stackoverflow.com/questions/5929107/python-decorators-with-parameters
    :param dec:
    :return:
    """

    def layer(*args, **kwargs):
        def repl(f):
            return dec(f, *args, **kwargs)
        return repl
    return layer


@parametrized
def temporary_file(func, extension, delete_on_exit=True):
    """
    This method decorator creates a filename with date using the provided extension and delete the file after the method
    has been executed.

    The settings.TEST_OUTPUT_PATH must be configured in your settings file.

    .. code-block:: python

        @temporary_file('json')
        def test_temporary_file_decorator(self):
            filename = self.test_temporary_file_decorator.filename
            ... write to the file ...


    :param func: function to decorate
    :param extension: extension of the filename without the dot
    :param delete_on_exit: If True the filename will be deleted.
    :return: the function
    """
    filename = create_dated('{}.{}'.format(func.__name__, extension))

    def function_t_return(*args):
        results = func(*args)
        if os.path.exists(filename) and delete_on_exit:
            os.remove(filename)
        return results

    function_t_return.filename = filename
    return function_t_return


def json_serial(obj):
    """JSON serializer for objects not serializable by default json code
    taken from: https://stackoverflow.com/questions/11875770/how-to-overcome-datetime-datetime-not-json-serializable
    """

    if isinstance(obj, (datetime, date)):
        serial = obj.isoformat()
        return serial
    raise TypeError("Type %s not serializable" % type(obj))


def serialize_data(data, output_file=None, format='json', **kwargs):
    """
    Quick function to serialize a data to file. The data keys will be saved in an alphabetical order
    for consistency purposes.
    If no output_file is supplied the function will created a dated file in the settings.TEST_OUTPUT_PATH folder.
    if the output_file is a folder the dated file will be created on the supplied folder with the serialized date.
    if the output_file is a file the data will be serialized to thar file

    :param data: Dictionary or list to serialize
    :param format: Format to serialize to. Currently json is the only one supported
    :param output_file: File to output the data to
    :param kwargs:
    """
    assert format in ['json', 'pickle'], 'Unsupported format {}'.format(format)
    base_filename = kwargs.get('base_filename', 'serialized_data')

    if output_file is None:
        filename = create_dated('{}.{}'.format(base_filename, format))
    elif os.path.isdir(output_file):
        filename = os.path.join(output_file, '{}.{}'.format(base_filename, format))
    else:
{'fraction_non_alphanumeric': 0.08630676552363299, 'fraction_numerical': 0.005097312326227989, 'mean_word_length': 3.2443461160275318, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '101864', 'n_tokens_mistral': 2444, 'n_tokens_neox': 2377, 'n_words': 857}