<commit_before>import os
import copy
import tempfile
import unittest
import hashlib
import pyjsonrpc
import binascii


DEFAULT_RPC_URL = "http://127.0.0.1:7000"
STORJLIB_RPC_URL = os.environ.get("STORJLIB_RPC_URL", DEFAULT_RPC_URL)


def h(hexdata):
    data = binascii.unhexlify(hexdata)
    digest = hashlib.new("ripemd160", hashlib.sha256(data).digest()).digest()
    return binascii.hexlify(digest)


NUMCHALLENGES = 5
SHARD_DATA = os.urandom(1024)  # 1M
SHARD_HEXDATA = binascii.hexlify(SHARD_DATA)
SHARD_ID = h(SHARD_HEXDATA)
CHALLENGES = [binascii.hexlify(os.urandom(32)) for n in range(NUMCHALLENGES)]
R0 = h(CHALLENGES[0] + SHARD_HEXDATA)
R1 = h(CHALLENGES[1] + SHARD_HEXDATA)
R2 = h(CHALLENGES[2] + SHARD_HEXDATA)
R3 = h(CHALLENGES[3] + SHARD_HEXDATA)
R4 = h(CHALLENGES[4] + SHARD_HEXDATA)
R5 = ""
R6 = ""
R7 = ""
LEAVES = [h(R0), h(R1), h(R2), h(R3), h(R4), h(R5), h(R6), h(R7)]
L0, L1, L2, L3, L4, L5, L6, L7 = LEAVES
ROOT = h(h(h(L0 + L1) + h(L2 + L3)) + h(h(L4 + L5) + h(L6 + L7)))
CHALLENGE = CHALLENGES[3]
N01 = h(L0 + L1)
N4567 = h(h(L4 + L5) + h(L6 + L7))
PROOF = [[N01, [L2, [R3]]], N4567]


@unittest.skip("not implemented")
class TestPrepare(unittest.TestCase):
    pass


class _Abs(object):

    def setUp(self):
        self.rpc = pyjsonrpc.HttpClient(url=STORJLIB_RPC_URL)

        # save temp shard
        shard_path = tempfile.mktemp()
        with open(shard_path, "wb") as shard:
            shard.write(SHARD_DATA)

        self.shardid = self.rpc.store_add(shard_path)  # import shard
        os.remove(shard_path)  # remove temp shard

    def tearDown(self):
        self.rpc.store_remove(self.shardid)  # remove temp shard from store


class TestPerform(_Abs, unittest.TestCase):

    def test_preform(self):
        self.assertEqual(self.shardid, SHARD_ID)
        proof = self.rpc.audit_perform(SHARD_ID, LEAVES, CHALLENGE)
        self.assertEqual(proof, PROOF)

    # TODO test invalid input

<commit_msg>fixed tests<commit_after>import os
import copy
import tempfile
import unittest
import hashlib
import pyjsonrpc
import binascii


DEFAULT_RPC_URL = "http://127.0.0.1:7000"
STORJLIB_RPC_URL = os.environ.get("STORJLIB_RPC_URL", DEFAULT_RPC_URL)


def h(hexdata):
    data = binascii.unhexlify(hexdata)
    digest = hashlib.new("ripemd160", hashlib.sha256(data).digest()).digest()
    return binascii.hexlify(digest)


NUMCHALLENGES = 5
SHARD_DATA = os.urandom(1024)  # 1M
SHARD_HEXDATA = binascii.hexlify(SHARD_DATA)
SHARD_ID = h(SHARD_HEXDATA)
CHALLENGES = [binascii.hexlify(os.urandom(32)) for n in range(NUMCHALLENGES)]
R0 = h(CHALLENGES[0] + SHARD_HEXDATA)
R1 = h(CHALLENGES[1] + SHARD_HEXDATA)
R2 = h(CHALLENGES[2] + SHARD_HEXDATA)
R3 = h(CHALLENGES[3] + SHARD_HEXDATA)
R4 = h(CHALLENGES[4] + SHARD_HEXDATA)
R5 = ""
R6 = ""
R7 = ""
LEAVES = [h(R0), h(R1), h(R2), h(R3), h(R4)]
ALL_LEAVES = [h(R0), h(R1), h(R2), h(R3), h(R4), h(R5), h(R6), h(R7)]
L0, L1, L2, L3, L4, L5, L6, L7 = ALL_LEAVES
ROOT = h(h(h(L0 + L1) + h(L2 + L3)) + h(h(L4 + L5) + h(L6 + L7)))
CHALLENGE = CHALLENGES[3]
N01 = h(L0 + L1)
N4567 = h(h(L4 + L5) + h(L6 + L7))
PROOF = [[N01, [L2, [R3]]], N4567]


class _Abs(object):

    def setUp(self):
        self.rpc = pyjsonrpc.HttpClient(url=STORJLIB_RPC_URL)

        # save temp shard
        shard_path = tempfile.mktemp()
        with open(shard_path, "wb") as shard:
            shard.write(SHARD_DATA)

        self.shardid = self.rpc.store_add(shard_path)  # import shard
        os.remove(shard_path)  # remove temp shard

    def tearDown(self):
        self.rpc.store_remove(self.shardid)  # remove temp shard from store


class TestPrepare(_Abs, unittest.TestCase):

    def test_prepare(self):
        self.assertEqual(self.shardid, SHARD_ID)
        leaves = self.rpc.audit_prepare(SHARD_ID, CHALLENGES)
        self.assertEqual(leaves, LEAVES)


class TestPerform(_Abs, unittest.TestCase):

    def test_preform(self):
        self.assertEqual(self.shardid, SHARD_ID)
        proof = self.rpc.audit_perform(SHARD_ID, LEAVES, CHALLENGE)
        self.assertEqual(proof, PROOF)

    # TODO test invalid input


{'fraction_non_alphanumeric': 0.14254224834680382, 'fraction_numerical': 0.04139113397012001, 'mean_word_length': 3.8968824940047964, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 2, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2935802', 'n_tokens_mistral': 1842, 'n_tokens_neox': 1745, 'n_words': 330}