<commit_before>import os, tempfile, json, pickle
import param
from lancet.core import PrettyPrinted

try:
    from io import StringIO
except:
    from StringIO import StringIO

try:    import numpy
except: pass

try:    import Image
except: pass

class FileType(PrettyPrinted, param.Parameterized):
    """
    The base class for all supported file types in Lancet. This class
    is designed to be simple and easily extensible to support new
    files and has only three essential methods: 'save', 'data' and
    'metadata').

    Optionally, a 'display' classmethod may be implemented to
    allow the file contents to be quickly visualized.
    """

    hash_suffix = param.Boolean(default=True, doc='''
       Whether to ensure the saved filename is unique by adding a
       short hash suffix. Note that this is a class level parameter
       only.''')

    directory = param.String(default='.', allow_None=True, doc='''
       Directory in which to load or save the file. Note that this
       is a class level parameter only.''')

    extensions = param.List(default=[], constant=True,
       doc= '''The set of supported file extensions.''')

    data_key = param.String(default='data', doc='''
       The name (key) given to the file contents if the key cannot be
       determined from the file itself.''')

    def __init__(self, **kwargs):
        super(FileType, self).__init__(**kwargs)
        self._pprint_args = ([],[],None,{})
        self.pprint_args(['data_key', 'hash_suffix'], [])

    def save(self, filename,  metadata={}, **data):
        """
        The implementation in the base class simply checks there is no
        clash between the metadata and data keys.
        """
        intersection = set(metadata.keys()) & set(data.keys())
        if intersection:
            msg = 'Key(s) overlap between data and metadata: %s'
            raise Exception(msg  % ','.join(intersection))

    def metadata(self, filename):
        """
        The metadata returned as a dictionary.
        """
        raise NotImplementedError

    def data(self, filename):
        """
        Data returned as a dictionary.
        """
        raise NotImplementedError

    def _loadpath(self, filename):
        (_, ext) = os.path.splitext(filename)
        if ext not in self.extensions:
            raise Exception("Unsupported extensions")
        abspath = os.path.abspath(os.path.join(self.directory, filename))
        return filename if os.path.isfile(filename) else abspath

    def _savepath(self, filename):
        """
        Returns the full path for saving the file, adding an extension
        and making the filename unique as necessary.
        """
        (basename, ext) = os.path.splitext(filename)
        ext = ext if ext else self.extensions[0]
        savepath = os.path.abspath(os.path.join(self.directory,
                                                 '%s%s' % (basename, ext)))
        return (tempfile.mkstemp(ext, basename + "_", self.directory)[1]
                if self.hash_suffix else savepath)

    @classmethod
    def file_supported(cls, filename):
        """
        Returns a boolean indicating whether the filename has an
        appropriate extension for this class.
        """
        if not isinstance(filename, str):
            return False<commit_msg>Improved how the _savepath method of FileType selects a file extension<commit_after>import os, tempfile, json, pickle
import param
from lancet.core import PrettyPrinted

try:
    from io import StringIO
except:
    from StringIO import StringIO

try:    import numpy
except: pass

try:    import Image
except: pass

class FileType(PrettyPrinted, param.Parameterized):
    """
    The base class for all supported file types in Lancet. This class
    is designed to be simple and easily extensible to support new
    files and has only three essential methods: 'save', 'data' and
    'metadata').

    Optionally, a 'display' classmethod may be implemented to
    allow the file contents to be quickly visualized.
    """

    hash_suffix = param.Boolean(default=True, doc='''
       Whether to ensure the saved filename is unique by adding a
       short hash suffix. Note that this is a class level parameter
       only.''')

    directory = param.String(default='.', allow_None=True, doc='''
       Directory in which to load or save the file. Note that this
       is a class level parameter only.''')

    extensions = param.List(default=[], constant=True,
       doc= '''The set of supported file extensions.''')

    data_key = param.String(default='data', doc='''
       The name (key) given to the file contents if the key cannot be
       determined from the file itself.''')

    def __init__(self, **kwargs):
        super(FileType, self).__init__(**kwargs)
        self._pprint_args = ([],[],None,{})
        self.pprint_args(['data_key', 'hash_suffix'], [])

    def save(self, filename,  metadata={}, **data):
        """
        The implementation in the base class simply checks there is no
        clash between the metadata and data keys.
        """
        intersection = set(metadata.keys()) & set(data.keys())
        if intersection:
            msg = 'Key(s) overlap between data and metadata: %s'
            raise Exception(msg  % ','.join(intersection))

    def metadata(self, filename):
        """
        The metadata returned as a dictionary.
        """
        raise NotImplementedError

    def data(self, filename):
        """
        Data returned as a dictionary.
        """
        raise NotImplementedError

    def _loadpath(self, filename):
        (_, ext) = os.path.splitext(filename)
        if ext not in self.extensions:
            raise Exception("Unsupported extensions")
        abspath = os.path.abspath(os.path.join(self.directory, filename))
        return filename if os.path.isfile(filename) else abspath

    def _savepath(self, filename):
        """
        Returns the full path for saving the file, adding an extension
        and making the filename unique as necessary.
        """
        (basename, ext) = os.path.splitext(filename)
        basename = basename if (ext in self.extensions) else filename
        ext = ext if (ext in self.extensions) else self.extensions[0]
        savepath = os.path.abspath(os.path.join(self.directory,
                                                 '%s%s' % (basename, ext)))
        return (tempfile.mkstemp(ext, basename + "_", self.directory)[1]
                if self.hash_suffix else savepath)

    @classmethod
    def file_supported(cls, filename):
        """
        Returns a boolean indicating whether the filename has an
        appropriate extension for this class.
        """
        if not isinstance(filename, str):
            return False
{'fraction_non_alphanumeric': 0.09693124815579816, 'fraction_numerical': 0.0005901445854234287, 'mean_word_length': 2.5015495867768593, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4469425', 'n_tokens_mistral': 1802, 'n_tokens_neox': 1757, 'n_words': 663}