<commit_before>import json
from urllib3 import ProxyManager, make_headers
from traptor import settings

USERS_LOOKUP_STR = settings.TWITTER_API_URL + '/users/lookup.json?screen_name={}'
SEARCH_TWEETS_STR = settings.TWITTER_API_URL + '/search/tweets.json?q={}&result_type=recent&count=100&include_entities=false'

def get_userid_for_username(username):
    http = ProxyManager(settings.PROXY_URL, proxy_headers=make_headers(proxy_basic_auth='{}:{}'.format(settings.PROXY_USER, settings.PROXY_PASSWORD)))
    resp = http.request('GET', USERS_LOOKUP_STR.format(username), timeout=settings.PROXY_TIMEOUT)
    assert resp.status == 200, 'Twitter API error ({}): {}'.format(resp.status, resp.data)
    data = json.loads(resp.data)
    return data[0]['id_str']

def get_recent_tweets_by_keyword(keyword):
    http = ProxyManager(settings.PROXY_URL, proxy_headers=make_headers(proxy_basic_auth='{}:{}'.format(settings.PROXY_USER, settings.PROXY_PASSWORD)))
    resp = http.request('GET', SEARCH_TWEETS_STR.format(keyword), timeout=settings.PROXY_TIMEOUT)
    assert resp.status == 200, 'Twitter API error ({}): {}'.format(resp.status, resp.data)
    data = json.loads(resp.data)
    return data<commit_msg>encode fields in piscina client<commit_after>import json
from urllib3 import ProxyManager, make_headers
from traptor import settings

USERS_LOOKUP_STR = settings.TWITTER_API_URL + '/users/lookup.json'
SEARCH_TWEETS_STR = settings.TWITTER_API_URL + '/search/tweets.json'

def _get_proxy():
    return ProxyManager(settings.PROXY_URL, proxy_headers=make_headers(proxy_basic_auth='{}:{}'.format(settings.PROXY_USER, settings.PROXY_PASSWORD)))

def get_userid_for_username(username):
    http = _get_proxy()

    fields = {
        'screen_name': username,
    }

    resp = http.request('GET', USERS_LOOKUP_STR, fields, timeout=settings.PROXY_TIMEOUT)
    assert resp.status == 200, 'Twitter API error ({}): {}'.format(resp.status, resp.data)
    data = json.loads(resp.data)
    return data[0]['id_str']

def get_recent_tweets_by_keyword(keyword):
    http = _get_proxy()

    fields = {
        'q': keyword,
        'result_type': 'recent',
        'count': 100,
        'include_entities': 'false'
    }

    resp = http.request('GET', SEARCH_TWEETS_STR, fields, timeout=settings.PROXY_TIMEOUT)
    assert resp.status == 200, 'Twitter API error ({}): {}'.format(resp.status, resp.data)
    data = json.loads(resp.data)
    return data
{'fraction_non_alphanumeric': 0.13357994245787094, 'fraction_numerical': 0.009042334566378957, 'mean_word_length': 5.27319587628866, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6942350', 'n_tokens_mistral': 854, 'n_tokens_neox': 840, 'n_words': 141}