<commit_before>__author__ = 'bobalot'

import struct
import io
import hashlib
from binascii import hexlify, unhexlify
from hashlib import sha256
import models

from error_code import error_code, obelisk_exception

# Py3 compatibility
import sys
bchr = chr
if sys.version > '3':
    bchr = lambda x: bytes([x])


def deser_uint32(f):
    if type(f) is not io.BytesIO:
        f = io.BytesIO(f)

    return struct.unpack(b"<I", f.read(4))[0]


def ser_uint32(u):
    rs = b""
    rs += struct.pack(b"<I", u & 0xFFFFFFFF)
    return rs


def deser_string(f):
    nit = struct.unpack(b"<B", f.read(1))[0]
    if nit == 253:
        nit = struct.unpack(b"<H", f.read(2))[0]
    elif nit == 254:
        nit = struct.unpack(b"<I", f.read(4))[0]
    elif nit == 255:
        nit = struct.unpack(b"<Q", f.read(8))[0]
    return f.read(nit)

def ser_string(s):
    if len(s) < 253:
        return bchr(len(s)) + s
    elif len(s) < 0x10000:
        return bchr(253) + struct.pack(b"<H", len(s)) + s
    elif len(s) < 0x100000000:
        return bchr(254) + struct.pack(b"<I", len(s)) + s
    return bchr(255) + struct.pack(b"<Q", len(s)) + s

def deser_hash(f):
    return f.read(32)

def ser_hash(h):
    return h[::-1]

def deser_uint256(f):
    r = 0
    for i in range(8):
        t = struct.unpack(b"<I", f.read(4))[0]
        r += t << (i * 32)
    return r

def ser_uint256(u):
    rs = b""
    for i in range(8):
        rs += struct.pack(b"<I", u & 0xFFFFFFFF)
        u >>= 32
    return rs

def ser_uint160(u):
    rs = b""
    for i in range(5):
        rs += struct.pack(b"<I", u & 0xFFFFFFFF)
        u >>= 32
    return rs

def uint160_from_str(s):
    r = 0
    t = struct.unpack(b"<IIIII", s[:20])
    for i in range(5):
        r += t[i] << (i * 32)
    return r

def uint256_from_str(s):
    r = 0
    t = struct.unpack(b"<IIIIIIII", s[:32])
    for i in range(8):
        r += t[i] << (i * 32)
    return r

def uint256_from_compact(c):
    nbytes = (c >> 24) & 0xFF
    v = (c & 0xFFFFFF) << (8 * (nbytes - 3))
    return v

def uint256_to_shortstr(u):
    s = "%064x" % (u,)
    return s[:16]

def deser_variable_uint(f):
    length = struct.unpack(b"<B", f.read(1))[0]
    if length < 0xfd:
        return length
    elif length == 0xfd:
        return struct.unpack(b"<H", f.read(2))[0]
    elif length == 0xfe:
        return struct.unpack(b"<I", f.read(4))[0]
    return struct.unpack(b"<Q", f.read(8))[0]

def deser_vector(f, c, arg1=None):
    count = deser_variable_uint(f)
    r = []
    for i in range(count):
        #if arg1 is not None:
        #    t = c(arg1)
        #else:
        #    t = c()

        if c is models.TxIn:
            t = deser_txin(f)
        elif c is models.TxOut:
            t = deser_txout(f)
        elif c is models.Transaction:
            t = deser_tx(f)
        else:
            raise NotImplementedError

        r.append(t)
    return r

def ser_vector(l):
    r = b""
    if len(l) < 253:
        r = bchr(len(l))
    elif len(l) < 0x10000:
        r = bchr(253) + struct.pack(b"<H", len(l))
    elif len(l) < 0x100000000:
        r = bchr(254) + struct.pack(b"<I", len(l))
    else:
        r = bchr(255) + struct.pack(b"<Q", len(l))
    for i in l:

        if type(i) is models.TxIn:
            r += ser_txin(i)
        elif type(i) is models.TxOut:
            r += ser_txout(i)
        elif type(i) is models.Transaction:
            r += ser_tx(i)
        else:
            raise NotImplementedError

    return r

def deser_uint256_vector(f):
    nit = struct.unpack(b"<B", f.read(1))[0]
    if nit == 253:
        nit = struct.unpack(b"<H", f.read(2))[0]
    elif nit == 254:
        nit = struct.unpack(b"<I", f.read(4))[0]
    elif nit == 255:
        nit = struct.unpack(b"<Q", f.read(8))[0]
    r = []
    for i in range(nit):
        t = deser_uint256(f)
        r.append(t)
    return r

def ser_uint256_vector(l):
    r = b""
    if len(l) < 253:
        r = bchr(len(l))
    elif len(s) < 0x10000:
        r = bchr(253) + struct.pack(b"<H", len(l))
    elif len(s) < 0x100000000:
        r = bchr(254) + struct.pack(b"<I", len(l))
    else:
        r = bchr(255) + struct.pack(b"<Q", len(l))
    for i in l:
        r += ser_uint256(i)
    return r

def deser_string_vector(f):
    nit = struct.unpack(b"<B", f.read(1))[0]
    if nit == 253:
        nit = struct.unpack(b"<H", f.read(2))[0]
    elif nit == 254:
        nit = struct.unpack(b"<I", f.read(4))[0]
    elif nit == 255:
        nit = struct.unpack(b"<Q", f.read(8))[0]
    r = []
    for i in range(nit):
        t = deser_string(f)
        r.append(t)
    return r

def ser_string_vector(l):
    r = b""
    if len(l) < 253:
        r = bchr(len(l))
    elif len(s) < 0x10000:
        r = bchr(253) + struct.pack(b"<H", len(l))
    elif len(s) < 0x100000000:
        r = bchr(254) + struct.pack(b"<I", len(l))
    else:
        r = bchr(255) + struct.pack(b"<Q", len(l))
    for sv in l:
        r += ser_string(sv)
    return r

def deser_int_vector(f):
    nit = struct.unpack(b"<B", f.read(1))[0]
    if nit == 253:
        nit = struct.unpack(b"<H", f.read(2))[0]
    elif nit == 254:
        nit = struct.unpack(b"<I", f.read(4))[0]
    elif nit == 255:
        nit = struct.unpack(b"<Q", f.read(8))[0]
    r = []
    for i in range(nit):
        t = struct.unpack(b"<i", f.read(4))[0]
        r.append(t)
    return r

def ser_int_vector(l):
    r = b""
    if len(l) < 253:
        r = bchr(len(l))
    elif len(s) < 0x10000:
        r = bchr(253) + struct.pack(b"<H", len(l))
    elif len(s) < 0x100000000:
        r = bchr(254) + struct.pack(b"<I", len(l))
    else:
        r = bchr(255) + struct.pack(b"<Q", len(l))
    for i in l:
        r += struct.pack(b"<i", i)
    return r

def Hash(s):
    return uint256_from_str(hashlib.sha256(hashlib.sha256(s).digest()).digest())

def Hash160(s):
    h = hashlib.new('ripemd160')
    h.update(hashlib.sha256(s).digest())
    return uint160_from_str(h.digest())


def ser_destination(destination):
    if destination is None:
        serialized = ""
    else:
        try:
            serialized = unhexlify(destination)
        except TypeError:
            serialized = destination

    return serialized

def deser_txout(f):
    txout = models.TxOut()
    txout.value = struct.unpack(b"<q", f.read(8))[0]
    txout.script = deser_string(f)
    return txout

def ser_txout(txout):
    r = b""
    r += struct.pack(b"<q", txout.value)
    r += ser_string(txout.script)
    return r


def deser_output_point(f):
    if type(f) is not io.BytesIO:
        f = io.BytesIO(f)

    outpoint = models.OutPoint()
    outpoint.hash = deser_hash(f)
    outpoint.index = deser_uint32(f.read(4))
    return outpoint


def ser_output_point(outpoint):
    r = b""
    r += ser_hash(outpoint.hash)
    r += ser_uint32(outpoint.index)
    return r


def deser_txin(f):
    if type(f) is not io.BytesIO:
        f = io.BytesIO(f)

    txin = models.TxIn()
    txin.previous_output = deser_output_point(f)
    txin.script = deser_string(f)
    txin.sequence = deser_uint32(f)
    return txin

def ser_txin(txin):
    r = b""
    r += ser_output_point(txin.previous_output)
    r += ser_string(txin.script)
    r += ser_uint32(txin.sequence)
    return r

<commit_msg>serialization madness, reverse bytes order<commit_after>__author__ = 'bobalot'

import struct
import io
import hashlib
from binascii import hexlify, unhexlify
from hashlib import sha256
import models

from error_code import error_code, obelisk_exception

# Py3 compatibility
import sys
bchr = chr
if sys.version > '3':
    bchr = lambda x: bytes([x])


def deser_uint32(f):
    if type(f) is not io.BytesIO:
        f = io.BytesIO(f)

    return struct.unpack(b"<I", f.read(4))[0]


def ser_uint32(u):
    rs = b""
    rs += struct.pack(b"<I", u & 0xFFFFFFFF)
    return rs


def deser_string(f):
    nit = struct.unpack(b"<B", f.read(1))[0]
    if nit == 253:
        nit = struct.unpack(b"<H", f.read(2))[0]
    elif nit == 254:
        nit = struct.unpack(b"<I", f.read(4))[0]
    elif nit == 255:
        nit = struct.unpack(b"<Q", f.read(8))[0]
    return f.read(nit)

def ser_string(s):
    if len(s) < 253:
        return bchr(len(s)) + s
    elif len(s) < 0x10000:
        return bchr(253) + struct.pack(b"<H", len(s)) + s
    elif len(s) < 0x100000000:
        return bchr(254) + struct.pack(b"<I", len(s)) + s
    return bchr(255) + struct.pack(b"<Q", len(s)) + s

def deser_hash(f):
    return f.read(32)

def ser_hash(h):
    return h[::-1]

def deser_uint256(f):
    r = 0
    for i in range(8):
        t = struct.unpack(b"<I", f.read(4))[0]
        r += t << (i * 32)
    return r

def ser_uint256(u):
    rs = b""
    for i in range(8):
        rs += struct.pack(b"<I", u & 0xFFFFFFFF)
        u >>= 32
    return rs

def ser_uint160(u):
    rs = b""
    for i in range(5):
        rs += struct.pack(b"<I", u & 0xFFFFFFFF)
        u >>= 32
    return rs

def uint160_from_str(s):
    r = 0
    t = struct.unpack(b"<IIIII", s[:20])
    for i in range(5):
        r += t[i] << (i * 32)
    return r

def uint256_from_str(s):
    r = 0
    t = struct.unpack(b"<IIIIIIII", s[:32])
    for i in range(8):
        r += t[i] << (i * 32)
    return r

def uint256_from_compact(c):
    nbytes = (c >> 24) & 0xFF
    v = (c & 0xFFFFFF) << (8 * (nbytes - 3))
    return v

def uint256_to_shortstr(u):
    s = "%064x" % (u,)
    return s[:16]

def deser_variable_uint(f):
    length = struct.unpack(b"<B", f.read(1))[0]
    if length < 0xfd:
        return length
    elif length == 0xfd:
        return struct.unpack(b"<H", f.read(2))[0]
    elif length == 0xfe:
        return struct.unpack(b"<I", f.read(4))[0]
    return struct.unpack(b"<Q", f.read(8))[0]

def deser_vector(f, c, arg1=None):
    count = deser_variable_uint(f)
    r = []
    for i in range(count):
        #if arg1 is not None:
        #    t = c(arg1)
        #else:
        #    t = c()

        if c is models.TxIn:
            t = deser_txin(f)
        elif c is models.TxOut:
            t = deser_txout(f)
        elif c is models.Transaction:
            t = deser_tx(f)
        else:
            raise NotImplementedError

        r.append(t)
    return r

def ser_vector(l):
    r = b""
    if len(l) < 253:
        r = bchr(len(l))
    elif len(l) < 0x10000:
        r = bchr(253) + struct.pack(b"<H", len(l))
    elif len(l) < 0x100000000:
        r = bchr(254) + struct.pack(b"<I", len(l))
    else:
        r = bchr(255) + struct.pack(b"<Q", len(l))
    for i in l:

        if type(i) is models.TxIn:
            r += ser_txin(i)
        elif type(i) is models.TxOut:
            r += ser_txout(i)
        elif type(i) is models.Transaction:
            r += ser_tx(i)
        else:
            raise NotImplementedError

    return r

def deser_uint256_vector(f):
    nit = struct.unpack(b"<B", f.read(1))[0]
    if nit == 253:
        nit = struct.unpack(b"<H", f.read(2))[0]
    elif nit == 254:
        nit = struct.unpack(b"<I", f.read(4))[0]
    elif nit == 255:
        nit = struct.unpack(b"<Q", f.read(8))[0]
    r = []
    for i in range(nit):
        t = deser_uint256(f)
        r.append(t)
    return r

def ser_uint256_vector(l):
    r = b""
    if len(l) < 253:
        r = bchr(len(l))
    elif len(s) < 0x10000:
        r = bchr(253) + struct.pack(b"<H", len(l))
    elif len(s) < 0x100000000:
        r = bchr(254) + struct.pack(b"<I", len(l))
    else:
        r = bchr(255) + struct.pack(b"<Q", len(l))
    for i in l:
        r += ser_uint256(i)
    return r

def deser_string_vector(f):
    nit = struct.unpack(b"<B", f.read(1))[0]
    if nit == 253:
        nit = struct.unpack(b"<H", f.read(2))[0]
    elif nit == 254:
        nit = struct.unpack(b"<I", f.read(4))[0]
    elif nit == 255:
        nit = struct.unpack(b"<Q", f.read(8))[0]
    r = []
    for i in range(nit):
        t = deser_string(f)
        r.append(t)
    return r

def ser_string_vector(l):
    r = b""
    if len(l) < 253:
        r = bchr(len(l))
    elif len(s) < 0x10000:
        r = bchr(253) + struct.pack(b"<H", len(l))
    elif len(s) < 0x100000000:
        r = bchr(254) + struct.pack(b"<I", len(l))
    else:
        r = bchr(255) + struct.pack(b"<Q", len(l))
    for sv in l:
        r += ser_string(sv)
    return r

def deser_int_vector(f):
    nit = struct.unpack(b"<B", f.read(1))[0]
    if nit == 253:
        nit = struct.unpack(b"<H", f.read(2))[0]
    elif nit == 254:
        nit = struct.unpack(b"<I", f.read(4))[0]
    elif nit == 255:
        nit = struct.unpack(b"<Q", f.read(8))[0]
    r = []
    for i in range(nit):
        t = struct.unpack(b"<i", f.read(4))[0]
        r.append(t)
    return r

def ser_int_vector(l):
    r = b""
    if len(l) < 253:
        r = bchr(len(l))
    elif len(s) < 0x10000:
        r = bchr(253) + struct.pack(b"<H", len(l))
    elif len(s) < 0x100000000:
        r = bchr(254) + struct.pack(b"<I", len(l))
    else:
        r = bchr(255) + struct.pack(b"<Q", len(l))
    for i in l:
        r += struct.pack(b"<i", i)
    return r

def Hash(s):
    return uint256_from_str(hashlib.sha256(hashlib.sha256(s).digest()).digest())

def Hash160(s):
    h = hashlib.new('ripemd160')
    h.update(hashlib.sha256(s).digest())
    return uint160_from_str(h.digest())


def ser_destination(destination):
    if destination is None:
        serialized = ""
    else:
        try:
            serialized = unhexlify(destination)
        except TypeError:
            serialized = destination

    return serialized

def deser_txout(f):
    txout = models.TxOut()
    txout.value = struct.unpack(b"<q", f.read(8))[0]
    txout.script = deser_string(f)
    return txout

def ser_txout(txout):
    r = b""
    r += struct.pack(b"<q", txout.value)
    r += ser_string(txout.script)
    return r


def deser_output_point(f):
    if type(f) is not io.BytesIO:
        f = io.BytesIO(f)

    outpoint = models.OutPoint()
    outpoint.hash = deser_hash(f)[::-1]
    outpoint.index = deser_uint32(f.read(4))
    return outpoint


def ser_output_point(outpoint):
    r = b""
    r += ser_hash(outpoint.hash)
    r += ser_uint32(outpoint.index)
    return r


def deser_txin(f):
    if type(f) is not io.BytesIO:
        f = io.BytesIO(f)

    txin = models.TxIn()
    txin.previous_output = deser_output_point(f)
    txin.script = deser_string(f)
    txin.sequence = deser_uint32(f)
    return txin

def ser_txin(txin):
    r = b""
    r += ser_output_point(txin.previous_output)
    r += ser_string(txin.script)
    r += ser_uint32(txin.sequence)
    return r


{'fraction_non_alphanumeric': 0.16449858902883888, 'fraction_numerical': 0.047009429417028015, 'mean_word_length': 2.254928315412186, 'pattern_counts': {'":': 0, '<': 143, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 18, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4361122', 'n_tokens_mistral': 6374, 'n_tokens_neox': 5891, 'n_words': 1338}