<commit_before>import json 
import requests 

from filestore import files 
from filestore import config

class Files(object):
    def __init__(self, name=config.values['files']['name'],
                       uri=config.values['client']['uri'],
                       keep_local=config.values['client']['keep_local'],
                       requester=requests,
                       ):
        self._name = name
        self._keep_local = keep_local
        if not uri.endswith('/'):
            uri += '/'
        self._uri = uri

    def request(self, rtype, *args, **kwargs):
        func = getattr(self.requester, rtype)
        r = func(*args, **kwargs)
        content_type = r.headers.get('content-type', None)
        if content_type != 'application/json':
            raise Exception(
                    "content-type!=application/json got %s(%s) %s\n'%s'" %\
                        (content_type, r.status_code, r.url, r.text))
        if not r.ok:
            try:
                out = r.json()
            except Exception:
                out = {}
            etype = out.get('exception', 'Exception')
            eclass = getattr(builtins, etype, Exception)
            raise eclass(out.get('message', 'status: %s' % r.status_code))
        try:
            out = r.json()
        except Exception:
            raise Exception("Failed to decode response after a 200 response")
        return out

    def __len__(self):
        uri = "%s%s/length" % (self._uri, self._name)
        return self.request('get', uri)['length']

    _get_default = object()
    def get(self, hexdigest, d=self._get_default):
        try:
            return self[hexdigest]
        except KeyError:
            if d != self._get_default:
                return d
            raise

    def __getitem__(self, hexdigest):
        # try and get the file back locally first
        try:
            fs = files.Files(name=self._name)
            return fs[hexdigest]
        except KeyError:
            pass
        uri = "%s%s/" % (self._uri, self._name, hexdigest)
        return self.request('get', uri)['data']

        

    
    

<commit_msg>add put select and iter<commit_after>import json 
import requests 
import zlib

import filestore
from filestore import config

class FilesClient(object):
    def __init__(self, name=config.values['files']['name'], 
                       uri=config.values['client']['uri'],
                       requester=requests,
                       ):
        """

        """
        self.requester = requester
        self._files = filestore.Files(name=name)
        self._name = name
        if not uri.endswith('/'):
            uri += '/'
        self._uri = uri

    def request(self, rtype, *args, **kwargs):
        func = getattr(self.requester, rtype)
        r = func(*args, **kwargs)
        content_type = r.headers.get('content-type', None)
        if content_type != 'application/json':
            raise Exception(
                    "content-type!=application/json got %s(%s) %s\n'%s'" %\
                        (content_type, r.status_code, r.url, r.text))
        if not r.ok:
            try:
                out = r.json()
            except Exception:
                out = {}
            etype = out.get('exception', 'Exception')
            eclass = getattr(builtins, etype, Exception)
            raise eclass(out.get('message', 'status: %s' % r.status_code))
        try:
            out = r.json()
        except Exception:
            raise Exception("Failed to decode response after a 200 response")
        return out

    def __len__(self):
        uri = "%s%s/length" % (self._uri, self._name)
        return self.request('get', uri)['length']

    _get_default = object()
    def get(self, hexdigest, d=self._get_default):
        try:
            return self[hexdigest]
        except KeyError:
            if d != self._get_default:
                return d
            return None

    def __getitem__(self, hexdigest):
        # try and get the file back locally first
        try:
            return self._files[hexdigest]
        except KeyError:
            pass
        # fetch data from server
        uri = "%s%s/file/%s" % (self._uri, self._name, hexdigest)
        data = zlib.decompress(self.request('get', uri)['data'])
        self._files[hexdigest] = data
        return self._files[hexdigest]

    def __setitem__(self, hexdigest, data):
        self.put(data, hexdigest=hexdigest)
        
    def put(self, data, hexdigest=None):
        self._files[hexdigest] = data
        uri = "%s%s/file/%s" % (self._uri, self._name, hexdigest)
        self.request('put', uri, data=zlib.compress(data))

    def select(self, a, b):
        uri = "%s%s/select/%s/%s" % self._uri, self._name, a, b)
        hexdigests = self.request('get', uri)['hexdigests']
        return hexdigests

    def __iter__(self):
        i = 0
        hexdigests = self.select(i, i + 10000)
        while hexdigests:
            for hexdigest in hexdigests:
                yield hexdigest
            i += 1
            hexdigests = self.select(i, i + 10000)

            

{'fraction_non_alphanumeric': 0.12109984399375975, 'fraction_numerical': 0.0035101404056162248, 'mean_word_length': 1.7413148049171565, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5174028', 'n_tokens_mistral': 1518, 'n_tokens_neox': 1440, 'n_words': 363}