<commit_before>__author__ = 'purg'

import abc
import hashlib
import logging
import mimetypes
import os
import os.path as osp
import tempfile

from smqtk.utils import safe_create_dir


MIMETYPES = mimetypes.MimeTypes()


class DataElement (object):
    """
    Base abstract class for a data element.

    Basic data elements have a UUID, some byte content, and a content type.

    DataElement implementations should be picklable for serialization, as some
    DataSet implementations will desire such functionality.

    """
    __metaclass__ = abc.ABCMeta

    def __init__(self):
        self._md5_cache = None
        self._temp_filepath_stack = []

    @property
    def _log(self):
        return logging.getLogger('.'.join([self.__module__,
                                           self.__class__.__name__]))

    def __hash__(self):
        return hash(self.uuid())

    def __del__(self):
        self.clean_temp()

    # TODO: __eq__/__ne__ methods?

    def md5(self):
        """
        :return: MD5 hex string of the data content.
        :rtype: str
        """
        if not self._md5_cache:
            self._md5_cache = hashlib.md5(self.get_bytes()).hexdigest()
        return self._md5_cache

    def write_temp(self, temp_dir=None):
        """
        Write this data's bytes to a temporary file on disk, returning the path
        to the written file, whose extension is guessed based on this data's
        content type.

        NOTE:
            The file path returned should not be explicitly removed by the user.
            Instead, the ``clean_temp()`` method should be called on this
            object.

        :param temp_dir: Optional directory to write temporary file in,
            otherwise we use the platform default temporary files directory.
            If this is an empty string, we count it the same as having provided
            None.
        :type temp_dir: None or str

        :return: Path to the temporary file
        :rtype: str

        """
        # Write a new temp file if there aren't any in the stack, or if the none
        # of the entries' base directory is the provided temp_dir (when one is
        # provided).

        def write_temp(d):
            """ Returns path to file written. Always creates new file. """
            if d:
                safe_create_dir(d)
            fd, fp = tempfile.mkstemp(
                suffix=MIMETYPES.guess_extension(self.content_type()),
                dir=d
            )
            os.close(fd)
            with open(fp, 'wb') as f:
                f.write(self.get_bytes())
            return fp

        if temp_dir:
            abs_temp_dir = osp.abspath(osp.expanduser(temp_dir))
            # Check if dir is the base of any path in the current stack.
            for tf in self._temp_filepath_stack:
                if osp.dirname(tf) == abs_temp_dir:
                    return tf
            # nothing in stack with given base directory, create new temp file
            self._temp_filepath_stack.append(write_temp(temp_dir))

        elif not self._temp_filepath_stack:
            # write new temp file to platform specific temp directory
            self._temp_filepath_stack.append(write_temp(None))

        # return last written temp file.
        return self._temp_filepath_stack[-1]

    def clean_temp(self):
        """
        Clean any temporary files created by this element. This does nothing if
        no temporary files have been generated for this element.
        """
        if len(self._temp_filepath_stack):
            for fp in self._temp_filepath_stack:
                if os.path.isfile(fp):<commit_msg>add writetemp suffix override for .jpe -> .jpg<commit_after>__author__ = 'purg'

import abc
import hashlib
import logging
import mimetypes
import os
import os.path as osp
import tempfile

from smqtk.utils import safe_create_dir


MIMETYPES = mimetypes.MimeTypes()


class DataElement (object):
    """
    Base abstract class for a data element.

    Basic data elements have a UUID, some byte content, and a content type.

    DataElement implementations should be picklable for serialization, as some
    DataSet implementations will desire such functionality.

    """
    __metaclass__ = abc.ABCMeta

    def __init__(self):
        self._md5_cache = None
        self._temp_filepath_stack = []

    @property
    def _log(self):
        return logging.getLogger('.'.join([self.__module__,
                                           self.__class__.__name__]))

    def __hash__(self):
        return hash(self.uuid())

    def __del__(self):
        self.clean_temp()

    # TODO: __eq__/__ne__ methods?

    def md5(self):
        """
        :return: MD5 hex string of the data content.
        :rtype: str
        """
        if not self._md5_cache:
            self._md5_cache = hashlib.md5(self.get_bytes()).hexdigest()
        return self._md5_cache

    def write_temp(self, temp_dir=None):
        """
        Write this data's bytes to a temporary file on disk, returning the path
        to the written file, whose extension is guessed based on this data's
        content type.

        NOTE:
            The file path returned should not be explicitly removed by the user.
            Instead, the ``clean_temp()`` method should be called on this
            object.

        :param temp_dir: Optional directory to write temporary file in,
            otherwise we use the platform default temporary files directory.
            If this is an empty string, we count it the same as having provided
            None.
        :type temp_dir: None or str

        :return: Path to the temporary file
        :rtype: str

        """
        # Write a new temp file if there aren't any in the stack, or if the none
        # of the entries' base directory is the provided temp_dir (when one is
        # provided).

        def write_temp(d):
            """ Returns path to file written. Always creates new file. """
            if d:
                safe_create_dir(d)
            ext = MIMETYPES.guess_extension(self.content_type())
            # Exceptions because mimetypes is apparently REALLY OLD
            if ext == '.jpe':
                ext = '.jpg'
            fd, fp = tempfile.mkstemp(
                suffix=ext,
                dir=d
            )
            os.close(fd)
            with open(fp, 'wb') as f:
                f.write(self.get_bytes())
            return fp

        if temp_dir:
            abs_temp_dir = osp.abspath(osp.expanduser(temp_dir))
            # Check if dir is the base of any path in the current stack.
            for tf in self._temp_filepath_stack:
                if osp.dirname(tf) == abs_temp_dir:
                    return tf
            # nothing in stack with given base directory, create new temp file
            self._temp_filepath_stack.append(write_temp(temp_dir))

        elif not self._temp_filepath_stack:
            # write new temp file to platform specific temp directory
            self._temp_filepath_stack.append(write_temp(None))

        # return last written temp file.
        return self._temp_filepath_stack[-1]

    def clean_temp(self):
        """
        Clean any temporary files created by this element. This does nothing if
        no temporary files have been generated for this element.
        """
        if len(self._temp_filepath_stack):
            for fp in self._temp_filepath_stack:
                if os.path.isfile(fp):
{'fraction_non_alphanumeric': 0.07056936647955092, 'fraction_numerical': 0.0021384656508954825, 'mean_word_length': 1.9682665608885364, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '666787', 'n_tokens_mistral': 2034, 'n_tokens_neox': 1970, 'n_words': 717}