<commit_before>#!/usr/bin/env python3
from __future__ import print_function
import json
import pickle
import requests
import traceback
from os import environ

try:
    from urllib.parse import urlencode
except ImportError:
    from urllib import urlencode

try:
    from misc.debug import TDB
    tdb=TDB()
    printD=tdb.printD
except ImportError:
    printD = print

# read environment variables # FIXME not the most modular...

api_token = environ.get('HYP_API_TOKEN', 'TOKEN')  # Hypothesis API token
username = environ.get('HYP_USERNAME', 'USERNAME') # Hypothesis username
group = environ.get('HYP_GROUP', '__world__')
print(api_token, username, group)  # sanity check

# annotation retrieval and memoization

class Memoizer:  # TODO the 'idea' solution to this is a self-updating list that listenes on the websocket and uses this transparently behind the scenes... yes there will be synchronization issues...
    def __init__(self, memoization_file, api_token=api_token, username=username, group=group):
        self.api_token = api_token
        self.username = username
        self.group = group
        self.memoization_file = memoization_file

    def __call__(self):
        return self.get_annos()

    def get_annos_from_api(self, offset=0, limit=None):
        print('yes we have to start from here')
        h = HypothesisUtils(username=self.username, token=self.api_token, group=self.group, max_results=100000)
        params = {'offset':offset,
                  'group':h.group}
        if limit is None:
            rows = h.search_all(params)
        else:
            params['limit'] = limit
            obj = h.search(params)
            rows = obj['rows']
            if 'replies' in obj:
                rows += obj['replies']
        annos = [HypothesisAnnotation(row) for row in rows]
        return annos

    def get_annos_from_file(self):<commit_msg>hypothesis.py memoizer if we are pulling from world always restrict to the user in question<commit_after>#!/usr/bin/env python3
from __future__ import print_function
import json
import pickle
import requests
import traceback
from os import environ

try:
    from urllib.parse import urlencode
except ImportError:
    from urllib import urlencode

try:
    from misc.debug import TDB
    tdb=TDB()
    printD=tdb.printD
except ImportError:
    printD = print

# read environment variables # FIXME not the most modular...

api_token = environ.get('HYP_API_TOKEN', 'TOKEN')  # Hypothesis API token
username = environ.get('HYP_USERNAME', 'USERNAME') # Hypothesis username
group = environ.get('HYP_GROUP', '__world__')
print(api_token, username, group)  # sanity check

# annotation retrieval and memoization

class Memoizer:  # TODO the 'idea' solution to this is a self-updating list that listenes on the websocket and uses this transparently behind the scenes... yes there will be synchronization issues...
    def __init__(self, memoization_file, api_token=api_token, username=username, group=group):
        self.api_token = api_token
        self.username = username
        self.group = group
        self.memoization_file = memoization_file

    def __call__(self):
        return self.get_annos()

    def get_annos_from_api(self, offset=0, limit=None):
        print('yes we have to start from here')
        h = HypothesisUtils(username=self.username, token=self.api_token, group=self.group, max_results=100000)
        params = {'offset':offset,
                  'group':h.group}
        if self.group == '__world__':
            params['user'] = self.username
        if limit is None:
            rows = h.search_all(params)
        else:
            params['limit'] = limit
            obj = h.search(params)
            rows = obj['rows']
            if 'replies' in obj:
                rows += obj['replies']
        annos = [HypothesisAnnotation(row) for row in rows]
        return annos

    def get_annos_from_file(self):
{'fraction_non_alphanumeric': 0.08854299566215872, 'fraction_numerical': 0.004082674151569278, 'mean_word_length': 3.0537745604963806, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5164775', 'n_tokens_mistral': 1171, 'n_tokens_neox': 1145, 'n_words': 352}