<commit_before>#!/usr/bin/env python3

import binascii
import datetime
import hashlib


def to_hex(bytestring):
    """
    convert given little-endian bytestring to hex
    """
    return binascii.hexlify(bytestring[::-1])


def double_hash(bytestring):
    """
    double SHA256 hash given bytestring
    """
    return hashlib.sha256(hashlib.sha256(bytestring).digest()).digest()


class BlockChain:

    def __init__(self, data):
        self.data = data
        self.index = 0
        self.block_count = 0

    def blocks(self):
        """
        yields blocks one at a time
        """
        while self.index < len(self.data):
            yield self.parse_block()
            self.block_count += 1

    def hash_since(self, mark):
        """
        double hash data from given mark to current index and return in hex
        """
        return to_hex(double_hash(self.data[mark:self.index]))


    ## basic reading of little-endian integers of different widths

    def get_uint8(self):
        data = self.data[self.index]
        self.index += 1
        return data

    def get_uint16(self):
        return self.get_uint8() + (self.get_uint8() << 8)

    def get_uint32(self):
        return self.get_uint16() + (self.get_uint16() << 16)

    def get_uint64(self):
        return self.get_uint32() + (self.get_uint32() << 32)


    ## more involved data reading

    def get_bytestring(self, length=1):
        data = self.data[self.index:self.index + length]
        self.index += length
        return data

    def get_timestamp(self):
        return datetime.datetime.utcfromtimestamp(self.get_uint32())

    def get_hash(self):
        return to_hex(self.get_bytestring(32))

    def get_varlen_int(self):
        code = self.get_uint8()
        if code < 0xFD:
            return code
        elif code == 0xFD:
            return self.get_uint16()
        elif code == 0xFE:
            return self.get_uint32()
        elif code == 0xFF:
            return self.get_uint64()

    def get_script(self):
        script_length = self.get_varlen_int()
        return self.get_bytestring(script_length)

<commit_msg>tokenization of scripts<commit_after>#!/usr/bin/env python3

import binascii
import datetime
import hashlib


def to_hex(bytestring):
    """
    convert given little-endian bytestring to hex
    """
    return binascii.hexlify(bytestring[::-1])


def double_hash(bytestring):
    """
    double SHA256 hash given bytestring
    """
    return hashlib.sha256(hashlib.sha256(bytestring).digest()).digest()


class BlockChain:

    def __init__(self, data):
        self.data = data
        self.index = 0
        self.block_count = 0

    def blocks(self):
        """
        yields blocks one at a time
        """
        while self.index < len(self.data):
            yield self.parse_block()
            self.block_count += 1

    def hash_since(self, mark):
        """
        double hash data from given mark to current index and return in hex
        """
        return to_hex(double_hash(self.data[mark:self.index]))


    ## basic reading of little-endian integers of different widths

    def get_uint8(self):
        data = self.data[self.index]
        self.index += 1
        return data

    def get_uint16(self):
        return self.get_uint8() + (self.get_uint8() << 8)

    def get_uint32(self):
        return self.get_uint16() + (self.get_uint16() << 16)

    def get_uint64(self):
        return self.get_uint32() + (self.get_uint32() << 32)


    ## more involved data reading

    def get_bytestring(self, length=1):
        data = self.data[self.index:self.index + length]
        self.index += length
        return data

    def get_timestamp(self):
        return datetime.datetime.utcfromtimestamp(self.get_uint32())

    def get_hash(self):
        return to_hex(self.get_bytestring(32))

    def get_varlen_int(self):
        code = self.get_uint8()
        if code < 0xFD:
            return code
        elif code == 0xFD:
            return self.get_uint16()
        elif code == 0xFE:
            return self.get_uint32()
        elif code == 0xFF:
            return self.get_uint64()

    def get_script(self):
        script_length = self.get_varlen_int()
        script = self.get_bytestring(script_length)

        script_tokens = []

        script_index = 0
        while script_index < script_length:
            op_code = script[script_index]
            script_index += 1

            if op_code <= 75:
                script_tokens.append(script[script_index:script_index + op_code])
                script_index += op_code
            elif op_code == 118:
                script_tokens.append("OP_DUP")
            elif op_code == 136:
                script_tokens.append("OP_EQUALVERIFY")
            elif op_code == 169:
                script_tokens.append("OP_HASH160")
            elif op_code == 172:
                script_tokens.append("OP_CHECKSIG")
            else:
                print("unknown opcode", op_code)
                raise ValueError

        return script_tokens

{'fraction_non_alphanumeric': 0.10177514792899409, 'fraction_numerical': 0.02465483234714004, 'mean_word_length': 2.134116192830655, 'pattern_counts': {'":': 0, '<': 21, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '291309', 'n_tokens_mistral': 1650, 'n_tokens_neox': 1510, 'n_words': 357}