<commit_before>"""
Datastore based on files written to and retrieved from a local filesystem.
"""

import os
import datetime
import shutil
import logging
import mimetypes
from subprocess import Popen
import warnings

from base import DataStore, DataKey, DataItem, IGNORE_DIGEST


class DataFile(DataItem):
    """A file-like object, that represents a file in a local filesystem."""
    # current implementation just for real files
    
    def __init__(self, path, store):
        self.path = path
        self.full_path = os.path.join(store.root, path)
        if os.path.exists(self.full_path):
            stats = os.stat(self.full_path)
            self.size = stats.st_size
        else:
            raise IOError("File %s does not exist" % self.full_path)
            #self.size = None
        self.name = os.path.basename(self.full_path)
        self.extension = os.path.splitext(self.full_path)
        self.mimetype, self.encoding = mimetypes.guess_type(self.full_path)
    
    def get_content(self, max_length=None):
        f = open(self.full_path, 'r')
        if max_length:
            content = f.read(max_length)
        else:
            content = f.read()
        f.close()
        return content
    content = property(fget=get_content)
    
    @property
    def sorted_content(self):
        sorted_path = "%s,sorted" % self.full_path
        if not os.path.exists(sorted_path):
            cmd = "sort %s > %s" % (self.full_path, sorted_path)
            job = Popen(cmd, shell=True)
            job.wait()
        f = open(sorted_path, 'r')
        content = f.read()
        f.close()
        if len(content) != self.size: # sort adds a \n if the file does not end with one
            assert len(content) == self.size + 1
            content = content[:-1]
        return content
    

class FileSystemDataStore(DataStore):
    """
    Represents a locally-mounted filesystem. The root of the data store will
    generally be a subdirectory of the real filesystem.
    """
    data_item_class = DataFile
    
    def __init__(self, root):
        self.root = root or "./Data"

    def __str__(self):
        return self.root
    
    def __getstate__(self):
        return {'root': self.root}
    
    def __setstate__(self, state):
        self.__init__(**state)
    
    def __get_root(self):
        return self._root
    def __set_root(self, value):
        if not isinstance(value, basestring):
            raise TypeError("root must be a string")
        self._root = os.path.abspath(value)
        if not os.path.exists(self._root):
            os.makedirs(self._root)
    root = property(fget=__get_root, fset=__set_root)
    
    def _find_new_data_files(self, timestamp, ignoredirs=[".smt", ".hg", ".svn", ".git", ".bzr"]):
        """Finds newly created/changed files in dataroot."""
        # The timestamp-based approach creates problems when running several
        # experiments at once, since datafiles created by other experiments may
        # be mixed in with this one.
        # For this reason, concurrently running computations should each use
        # their own datastore, each with a different root.
        timestamp = timestamp.replace(microsecond=0) # Round down to the nearest second
        # Find and add new data files
        length_dataroot = len(self.root) + len(os.path.sep)
        new_files = []
        for root, dirs, files in os.walk(self.root):
            for igdir in ignoredirs:
                if igdir in dirs:
                    dirs.remove(igdir)
            for file in files:
                full_path = os.path.join(root, file)
                relative_path = os.path.join(root[length_dataroot:],file)
                last_modified = datetime.datetime.fromtimestamp(os.stat(full_path).st_mtime)
                if  last_modified >= timestamp:
                    new_files.append(relative_path)
        return new_files
    
    def find_new_data(self, timestamp):<commit_msg>We no longer take the absolute path of `DataStore` root, hence allowing the user the choice of whether to use absolute or relative paths (in particular, relative paths make projects easier to move and easier to access from other locations (e.g. with symbolic links or NFS - see ticket:2)<commit_after>"""
Datastore based on files written to and retrieved from a local filesystem.
"""

import os
import datetime
import shutil
import logging
import mimetypes
from subprocess import Popen
import warnings

from base import DataStore, DataKey, DataItem, IGNORE_DIGEST


class DataFile(DataItem):
    """A file-like object, that represents a file in a local filesystem."""
    # current implementation just for real files
    
    def __init__(self, path, store):
        self.path = path
        self.full_path = os.path.join(store.root, path)
        if os.path.exists(self.full_path):
            stats = os.stat(self.full_path)
            self.size = stats.st_size
        else:
            raise IOError("File %s does not exist" % self.full_path)
            #self.size = None
        self.name = os.path.basename(self.full_path)
        self.extension = os.path.splitext(self.full_path)
        self.mimetype, self.encoding = mimetypes.guess_type(self.full_path)
    
    def get_content(self, max_length=None):
        f = open(self.full_path, 'r')
        if max_length:
            content = f.read(max_length)
        else:
            content = f.read()
        f.close()
        return content
    content = property(fget=get_content)
    
    @property
    def sorted_content(self):
        sorted_path = "%s,sorted" % self.full_path
        if not os.path.exists(sorted_path):
            cmd = "sort %s > %s" % (self.full_path, sorted_path)
            job = Popen(cmd, shell=True)
            job.wait()
        f = open(sorted_path, 'r')
        content = f.read()
        f.close()
        if len(content) != self.size: # sort adds a \n if the file does not end with one
            assert len(content) == self.size + 1
            content = content[:-1]
        return content
    

class FileSystemDataStore(DataStore):
    """
    Represents a locally-mounted filesystem. The root of the data store will
    generally be a subdirectory of the real filesystem.
    """
    data_item_class = DataFile
    
    def __init__(self, root):
        self.root = root or "./Data"

    def __str__(self):
        return self.root
    
    def __getstate__(self):
        return {'root': self.root}
    
    def __setstate__(self, state):
        self.__init__(**state)
    
    def __get_root(self):
        return self._root
    def __set_root(self, value):
        if not isinstance(value, basestring):
            raise TypeError("root must be a string")
        self._root = value
        if not os.path.exists(self._root):
            os.makedirs(self._root)
    root = property(fget=__get_root, fset=__set_root)
    
    def _find_new_data_files(self, timestamp, ignoredirs=[".smt", ".hg", ".svn", ".git", ".bzr"]):
        """Finds newly created/changed files in dataroot."""
        # The timestamp-based approach creates problems when running several
        # experiments at once, since datafiles created by other experiments may
        # be mixed in with this one.
        # For this reason, concurrently running computations should each use
        # their own datastore, each with a different root.
        timestamp = timestamp.replace(microsecond=0) # Round down to the nearest second
        # Find and add new data files
        length_dataroot = len(self.root) + len(os.path.sep)
        new_files = []
        for root, dirs, files in os.walk(self.root):
            for igdir in ignoredirs:
                if igdir in dirs:
                    dirs.remove(igdir)
            for file in files:
                full_path = os.path.join(root, file)
                relative_path = os.path.join(root[length_dataroot:],file)
                last_modified = datetime.datetime.fromtimestamp(os.stat(full_path).st_mtime)
                if  last_modified >= timestamp:
                    new_files.append(relative_path)
        return new_files
    
    def find_new_data(self, timestamp):
{'fraction_non_alphanumeric': 0.09366964613689237, 'fraction_numerical': 0.0008571078731480347, 'mean_word_length': 2.280321285140562, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5496442', 'n_tokens_mistral': 2395, 'n_tokens_neox': 2344, 'n_words': 706}