<commit_before>from hashlib import sha256

import json

from twisted.internet import defer, reactor
from twisted.web.client import Agent

from ooni.utils.net import BodyReceiver, StringProducer, Downloader

class InputFile(object):
    def __init__(self, id, name=None, description=None, 
                 version=None, author=None, date=None):
        self.id = id
        self.name = name
        self.description = description
        self.version = version
        self.author = author
        self.date = date

        self.file_path = None

    def verify(self):
        digest = os.path.basename(self.file_path)
        with open(self.file_path) as f:
            file_hash = sha256(f.read())
            return file_hash.hexdigest() == digest
        return False

class OONIBClient(object):
    def __init__(self, address):
        self.address = address
        self.agent = Agent(reactor)
        self.input_files = {}

    def _request(self, method, urn, genReceiver, bodyProducer=None):
        finished = defer.Deferred()

        uri = self.address + urn
        d = self.agent.request(method, uri, bodyProducer)

        @d.addCallback
        def callback(response):
            content_length = response.headers.getRawHeaders('content-length')
            response.deliverBody(genReceiver(finished, content_length))

        @d.addErrback
        def eb(err):
            finished.errback(err)

        return finished

    def queryBackend(self, method, urn, query=None):
        bodyProducer = None
        if query:
            bodyProducer = StringProducer(json.dumps(query), bodyProducer)
    
        def genReceiver(finished, content_length):
            return BodyReceiver(finished, content_length, json.loads)

        return self._request(method, urn, genReceiver, bodyProducer)

    def download(self, urn, download_path):

        def genReceiver(finished, content_length):
            return Downloader(download_path, finished, content_length)

        return self._request('GET', urn, genReceiver)
    
    def getNettestPolicy(self):
        pass

    def queryBouncer(self, requested_helpers):
        pass

    def getInput(self, input_hash):
        return self.queryBackend('GET', '/input/' + input_hash)

    def getInputList(self):
        return self.queryBackend('GET', '/input')

    def downloadInput(self, input_hash, download_path):
        return self.download('/input/'+input_hash+'/file', download_path)

    def getInputPolicy(self):
        pass
<commit_msg>Store a cache of input descriptor lookups<commit_after>from hashlib import sha256

import json

from twisted.internet import defer, reactor
from twisted.web.client import Agent

from ooni.utils.net import BodyReceiver, StringProducer, Downloader

class InputFile(object):
    def __init__(self, descriptor):
        self.id = descriptor['id']
        self.name = descriptor['name']
        self.version = descriptor['version']
        self.author = descriptor['author']
        self.date = descriptor['date']

        self.file_path = None

    def verify(self):
        digest = os.path.basename(self.file_path)
        with open(self.file_path) as f:
            file_hash = sha256(f.read())
            return file_hash.hexdigest() == digest
        return False

class OONIBClient(object):
    def __init__(self, address):
        self.address = address
        self.agent = Agent(reactor)
        self.input_files = {}

    def _request(self, method, urn, genReceiver, bodyProducer=None):
        finished = defer.Deferred()

        uri = self.address + urn
        d = self.agent.request(method, uri, bodyProducer)

        @d.addCallback
        def callback(response):
            content_length = response.headers.getRawHeaders('content-length')
            response.deliverBody(genReceiver(finished, content_length))

        @d.addErrback
        def eb(err):
            finished.errback(err)

        return finished

    def queryBackend(self, method, urn, query=None):
        bodyProducer = None
        if query:
            bodyProducer = StringProducer(json.dumps(query), bodyProducer)
    
        def genReceiver(finished, content_length):
            return BodyReceiver(finished, content_length, json.loads)

        return self._request(method, urn, genReceiver, bodyProducer)

    def download(self, urn, download_path):

        def genReceiver(finished, content_length):
            return Downloader(download_path, finished, content_length)

        return self._request('GET', urn, genReceiver)
    
    def getNettestPolicy(self):
        pass

    def queryBouncer(self, requested_helpers):
        pass

    def getInput(self, input_hash):
        try:
            return defer.succeed(self.input_files[input_hash])
        except KeyError:
            d = self.queryBackend('GET', '/input/' + input_hash)
            @d.addCallback
            def cb(descriptor):
                self.input_files[input_hash] = InputFile(descriptor)
                return self.input_files[input_hash]
            @d.addErrback
            def err(err):
                log.err("Failed to get descriptor for input %s" % input_hash)
                log.exception(err)
            return d

    def getInputList(self):
        return self.queryBackend('GET', '/input')

    def downloadInput(self, input_hash, download_path):
        return self.download('/input/'+input_hash+'/file', download_path)

    def getInputPolicy(self):
        pass

{'fraction_non_alphanumeric': 0.09546452084857351, 'fraction_numerical': 0.0021945866861741038, 'mean_word_length': 2.535229476405947, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 18, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5679871', 'n_tokens_mistral': 1548, 'n_tokens_neox': 1500, 'n_words': 357}