<commit_before># File abstraction to allow both S3 and local to be used

import os
import subprocess
import time

# for s3 from python
from boto.s3.connection import S3Connection

PROJECT_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))


class FileSystem(object):

    def read_file(self, path):
        raise NotImplemented

    def write_file(self, path, data):
        raise NotImplemented

    def list(self, path):
        raise NotImplemented

    def list_name_modified(self, path):
        raise NotImplemented

    def remove(self, path):
        raise NotImplemented


class S3FileSystem(FileSystem):

    def __init__(self, bucket_name, aws_access_key_id, aws_secret_access_key, security_token=None):
        conn = S3Connection(
            aws_access_key_id=aws_access_key_id,
            aws_secret_access_key=aws_secret_access_key,
            security_token=security_token)
        self.bucket = conn.get_bucket(bucket_name)

    def read_file(self, path):
        key = self.bucket.get_key(path)
        if key is None:
            return None
        return key.read()

    def write_file(self, path, data):
        key = self.bucket.get_key(path)
        if key is None:
            key = self.bucket.new_key(path)
        key.set_contents_from_string(data)

    def list(self, path=''):
        return [key.name for key in self.bucket.list(path)]

    def list_name_modified(self, path=''):
        model_names_and_modified = {}
        for key in self.bucket.list(path):
            model_names_and_modified[key.name] = key.last_modified
        return model_names_and_modified

    def remove(self, path):
        keys = self.bucket.get_all_keys(prefix=path)
        self.bucket.delete_keys(keys)


class LocalFileSystem(FileSystem):

    def __init__(self, basedir=None):
        if basedir is None:
            basedir = PROJECT_ROOT
        self.basedir = basedir

    def read_file(self, path):
        try:
            with open(self.abspath(path), 'r') as f:
                return f.read()
        except IOError:
            return None

    def write_file(self, path, data):
        # Make the intermediate directories if they don't exist<commit_msg>Experiment with moving all data into a "subdirectory" of an S3 bucket<commit_after># File abstraction to allow both S3 and local to be used

import os
import subprocess
import time

# for s3 from python
from boto.s3.connection import S3Connection

PROJECT_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))


class FileSystem(object):

    def read_file(self, path):
        raise NotImplemented

    def write_file(self, path, data):
        raise NotImplemented

    def list(self, path):
        raise NotImplemented

    def list_name_modified(self, path):
        raise NotImplemented

    def remove(self, path):
        raise NotImplemented


class S3FileSystem(FileSystem):

    def __init__(self,
                 bucket_name,
                 aws_access_key_id,
                 aws_secret_access_key,
                 security_token=None,
                 subdirectory=''):
        conn = S3Connection(
            aws_access_key_id=aws_access_key_id,
            aws_secret_access_key=aws_secret_access_key,
            security_token=security_token)
        self.bucket = conn.get_bucket(bucket_name)
        self.subdirectory = subdirectory
        if subdirectory and not subdirectory.endswith('/'):
            self.subdirectory += '/'

    def read_file(self, path):
        key = self.bucket.get_key(self.subdirectory + path)
        if key is None:
            return None
        return key.read()

    def write_file(self, path, data):
        key = self.bucket.get_key(self.subdirectory + path)
        if key is None:
            key = self.bucket.new_key(self.subdirectory + path)
        key.set_contents_from_string(data)

    def list(self, path=''):
        return [key.name for key in self.bucket.list(self.subdirectory + path)]

    def list_name_modified(self, path=''):
        model_names_and_modified = {}
        for key in self.bucket.list(self.subdirectory + path):
            model_names_and_modified[key.name] = key.last_modified
        return model_names_and_modified

    def remove(self, path):
        keys = self.bucket.get_all_keys(prefix=self.subdirectory + path)
        self.bucket.delete_keys(keys)


class LocalFileSystem(FileSystem):

    def __init__(self, basedir=None):
        if basedir is None:
            basedir = PROJECT_ROOT
        self.basedir = basedir

    def read_file(self, path):
        try:
            with open(self.abspath(path), 'r') as f:
                return f.read()
        except IOError:
            return None

    def write_file(self, path, data):
        # Make the intermediate directories if they don't exist
{'fraction_non_alphanumeric': 0.08367940199335548, 'fraction_numerical': 0.0026993355481727574, 'mean_word_length': 2.521198830409357, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4704500', 'n_tokens_mistral': 1450, 'n_tokens_neox': 1406, 'n_words': 353}