<commit_before>import json
import time
import requests


DELAY_INIT = 500
DELAY_MAX = 60 * 1000
DELAY_INC = 10
DELAY_MUL = 1.01


def _single_request(url, data):
    req = requests.post(url, headers={
        "Content-Type": "application/json",
    }, data=data)
    if req.status_code == 200:
        return json.loads(req.text)
    raise ValueError(
        "error {0} in worker request:\n{1}".format(req.status_code, req.text))


def worker_request(url, payload):
    done = False
    token = None
    try:
        res = _single_request(url, {
            "action": "start",
            "payload": payload,
        })
        delay = DELAY_INIT
        while not res["done"]:
            if not res["continue"]:
                raise ValueError("request has timed out")
            token = res["token"]
            time.sleep(delay / 1000.0)
            res = _single_request(url, {
                "action": "get",
                "token": token,
            })
            delay = min(max(delay * DELAY_MUL, delay + DELAY_INC), DELAY_MAX)
        if res["continue"]:
            cargo_tokens = res["result"]

            def check(ctoken, response):
                if response["token"] != ctoken:
                    raise ValueError("token mismatch {0} != {1}".format(
                        response["token"], ctoken))
                return response["result"]

            # TODO: async would be better
            return json.loads("".join(
                check(ctoken, _single_request(url, {
                    "action": "cargo",
                    "token": ctoken,
                }))
                for ctoken in cargo_tokens
            ))
        else:
            return json.loads(res["result"])
    finally:
        if not done and token is not None:
            while True:
                res = _single_request(url, {
                    "action": "stop",
                    "token": token,
                })
                if not res["continue"]:<commit_msg>even better<commit_after>import json
import time
import requests


DELAY_INIT = 500
DELAY_MAX = 60 * 1000
DELAY_INC = 10
DELAY_MUL = 1.01


def _single_request(url, data):
    req = requests.post(url, headers={
        "Content-Type": "application/json",
    }, data=data)
    if req.status_code == 200:
        return json.loads(req.text)
    raise ValueError(
        "error {0} in worker request:\n{1}".format(req.status_code, req.text))


def worker_request(url, payload):
    done = False
    token = None
    try:
        res = _single_request(url, {
            "action": "start",
            "payload": payload,
        })
        delay = DELAY_INIT
        while not res["done"]:
            if not res["continue"]:
                raise ValueError("request has timed out")
            token = res["token"]
            time.sleep(delay / 1000.0)
            res = _single_request(url, {
                "action": "get",
                "token": token,
            })
            delay = min(max(delay * DELAY_MUL, delay + DELAY_INC), DELAY_MAX)
        if res["continue"]:
            cargo_tokens = res["result"]

            def check(ctoken, response):
                if response["token"] != ctoken:
                    raise ValueError("token mismatch {0} != {1}".format(
                        response["token"], ctoken))
                return response["result"]

            # TODO: async would be better
            final = json.loads("".join(
                check(ctoken, _single_request(url, {
                    "action": "cargo",
                    "token": ctoken,
                }))
                for ctoken in cargo_tokens
            ))
        else:
            final = json.loads(res["result"])
        done = True
        return final
    finally:
        if not done and token is not None:
            while True:
                res = _single_request(url, {
                    "action": "stop",
                    "token": token,
                })
                if not res["continue"]:
{'fraction_non_alphanumeric': 0.11581569115815692, 'fraction_numerical': 0.012951432129514322, 'mean_word_length': 1.5084322298563397, 'pattern_counts': {'":': 18, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7224277', 'n_tokens_mistral': 1169, 'n_tokens_neox': 1086, 'n_words': 282}