<commit_before>#
# Common sqlchain support utils
#
import os, sys, pwd, time, json, threading, re, glob, urllib2, hashlib

from datetime import datetime
from struct import pack, unpack, unpack_from
from bitcoinrpc.authproxy import AuthServiceProxy, JSONRPCException

from backports.functools_lru_cache import lru_cache # pylint:disable=relative-import
from sqlchain.version import coincfg, ADDR_PREFIX, P2SH_PREFIX, P2SH_CHAR, BECH_HRP, BECH32_FLAG, BECH32_LONG, P2SH_FLAG
from sqlchain.version import S3_BLK_SIZE, BLOB_SPLIT_SIZE

tidylog = threading.Lock()

b58 = "123456789ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz"

class dotdict(dict):
    """dot.notation for more concise globals access on dict"""
    def __getattr__(self, attr):
        return self.get(attr)
    __setattr__= dict.__setitem__
    __delattr__= dict.__delitem__

# address support stuff
def is_address(addr):
    if addr[:2].lower() == coincfg(BECH_HRP):
        return bech32decode(addr) != None
    try:
        n = 0
        for c in addr:
            n = n * 58 + b58.index(c)
        btc = ('%%0%dx' % (25 << 1) % n).decode('hex')[-25:]
        return btc[-4:] == hashlib.sha256(hashlib.sha256(btc[:-4]).digest()).digest()[:4]
    except Exception:
        return False

def mkpkh(pk):
    rmd = hashlib.new('ripemd160')
    rmd.update(hashlib.sha256(pk).digest())
    return rmd.digest()

def addr2pkh(addr):
    if addr[:2].lower() == coincfg(BECH_HRP):
        pkh = bech32decode(addr)
        return str(pkh[2:]) if pkh else None
    long_value = 0L
    for (i, c) in enumerate(addr[::-1]):
        long_value += b58.find(c) * (58**i)
    result = ''
    while long_value >= 256:
        div, mod = divmod(long_value, 256)
        result = chr(mod) + result
        long_value = div
    result = chr(long_value) + result
    nPad = 0
    for c in addr:
        if c == b58[0]:
            nPad += 1
        else:
            break
    result = chr(0)*nPad + result
    return result[1:-4]

def mkaddr(pkh, aid=None, p2sh=False, bech32=False):
    if pkh == '\0'*20 and aid==0:
        return '' # pkh==0 id==0, special case for null address when op_return or non-std script
    if bech32 or (aid and (aid & BECH32_FLAG != 0)):
        return bech32encode(coincfg(BECH_HRP), pkh)
    pad = ''
    an = chr(coincfg(ADDR_PREFIX) if (aid is None and not p2sh) or (aid is not None and (aid & P2SH_FLAG != P2SH_FLAG)) else coincfg(P2SH_PREFIX)) + str(pkh)
    for c in an:
        if c == '\0':
            pad += '1'
        else:
            break
    num = long((an + hashlib.sha256(hashlib.sha256(an).digest()).digest()[0:4]).encode('hex'), 16)
    out = ''
    while num >= 58:
        num,m = divmod(num, 58)
        out = b58[m] + out
    return pad + b58[num] + out

def is_BL32(addr_id):
    return addr_id and (addr_id & BECH32_LONG) == BECH32_LONG

def addr2id(addr, cur=None, rtnPKH=False):
    pkh = addr2pkh(addr)
    if not pkh:
        return None,'' if rtnPKH else None
    addr_id, = unpack('<q', hashlib.sha256(pkh).digest()[:5]+'\0'*3)
    if addr[:2] == coincfg(BECH_HRP): # bech32 has bit 42 set, >20 byte, encode as odd id and stored in bech32 table
        addr_id |= BECH32_FLAG
        if len(pkh) > 20:
            addr_id |= P2SH_FLAG
    elif addr[0] in coincfg(P2SH_CHAR): # encode P2SH flag
        addr_id |= P2SH_FLAG
    if cur:
        tbl = 'bech32' if is_BL32(addr_id) else 'address'
        cur.execute("select id from {0} where id>=%s and id<%s+32 and addr=%s limit 1;".format(tbl), (addr_id,addr_id,pkh))
        row = cur.fetchone()
        return row[0] if row else None
    return addr_id,pkh if rtnPKH else addr_id

# script support stuff
def mkSPK(pkh, addr_id):
    if addr_id & BECH32_FLAG: # bech32 id have witness spk
        return ('\x16','\x00\x14%s'%pkh) if (addr_id & P2SH_FLAG != P2SH_FLAG) else ('\x22','\x00\x20%s'%pkh)
    return ('\x19','\x76\xa9\x14%s\x88\xac'%pkh) if (addr_id & P2SH_FLAG != P2SH_FLAG) else ('\x17','\xa9\x14%s\x87'%pkh)

def decodeScriptPK(data):
    result = { 'type':'other', 'data':data } # other, non-std
    if len(data) > 1:
        if len(data) == 25 and data[:3] == '\x76\xa9\x14' and data[23:25] == '\x88\xac': # P2PKH
            result = { 'type':'p2pkh', 'data':'', 'addr':mkaddr(data[3:23]) }
        elif len(data) == 23 and data[:2] == '\xa9\x14' and data[22] == '\x87': # P2SH
            result = { 'type':'p2sh', 'data':'', 'addr':mkaddr(data[2:22],p2sh=True)}
        elif len(data) == 67 and data[0] == '\x41' and data[66] == '\xac': # P2PK
            result = { 'type':'p2pk', 'data':data, 'addr':mkaddr(mkpkh(data[1:66])) }
        elif len(data) >= 35 and data[0] == '\x21' and data[34] == '\xac': # P2PK (compressed key)
            result = { 'type':'p2pk', 'data':data, 'addr':mkaddr(mkpkh(data[1:34])) }
        elif len(data) == 22 and data[:2] == '\x00\x14': # P2WPKH
            result = { 'type':'p2wpkh', 'data':'', 'addr':mkaddr(data[2:22],bech32=True) }
        elif len(data) == 34 and data[:2] == '\x00\x20': # P2WSH
            result = { 'type':'p2wsh', 'data':'', 'addr':mkaddr(data[2:34],bech32=True) }
        elif len(data) >= 38 and data[:6] == '\x6a\x24\xaa\x21\xa9\xed': # witness commitment
            result = { 'type':'witness', 'hash':data[6:38], 'data':data[38:] }
        elif len(data) <= 41 and data[0] == '\x6a': # OP_RETURN
            result = { 'type':'null', 'data':data }
    return result

OpCodes = { '\x4F':'OP_1NEGATE', '\x61':'OP_NOP', '\x63':'OP_IF', '\x64':'OP_NOTIF', '\x67':'OP_ELSE', '\x68':'OP_ENDIF',
            '\x69':'OP_VERIFY', '\x6A':'OP_RETURN', '\x6B':'OP_TOALTSTACK', '\x6C':'OP_FROMALTSTACK', '\x6D':'OP_2DROP', '\x6E':'OP_2DUP',
            '\x6F':'OP_3DUP', '\x70':'OP_2OVER', '\x71':'OP_2ROT', '\x72':'OP_2SWAP','\x73':'OP_IFDUP', '\x74':'OP_DEPTH', '\x75':'OP_DROP',
            '\x76':'OP_DUP', '\x77':'OP_NIP', '\x78':'OP_OVER', '\x79':'OP_PICK', '\x7A':'OP_ROLL', '\x7B':'OP_ROT', '\x7C':'OP_SWAP',
            '\x7D':'OP_TUCK',  '\x7E':'OP_CAT', '\x7F':'OP_SUBSTR', '\x80':'OP_LEFT', '\x81':'OP_RIGHT', '\x82':'OP_SIZE', '\x83':'OP_INVERT',
            '\x84':'OP_AND', '\x85':'OP_OR', '\x86':'OP_XOR', '\x87':'OP_EQUAL', '\x88':'OP_EQUALVERIFY', '\x8B':'OP_1ADD', '\x8C':'OP_1SUB',
            '\x8D':'OP_2MUL', '\x8E':'OP_2DIV', '\x8F':'OP_NEGATE', '\x90':'OP_ABS', '\x91':'OP_NOT', '\x92':'OP_0NOTEQUAL', '\x93':'OP_ADD',
            '\x94':'OP_SUB', '\x95':'OP_MUL', '\x96':'OP_DIV', '\x97':'OP_MOD', '\x98':'OP_LSHIFT', '\x99':'OP_RSHIFT', '\x9A':'OP_BOOLAND',
            '\x9B':'OP_BOOLOR', '\x9C':'OP_NUMEQUAL', '\x9D':'OP_NUMEQUALVERIFY', '\x9E':'OP_NUMNOTEQUAL', '\x9F':'OP_LESSTHAN',
            '\xA0':'OP_GREATERTHAN', '\xA1':'OP_LESSTHANOREQUAL', '\xA2':'OP_GREATERTHANOREQUAL', '\xA3':'OP_MIN', '\xA4':'OP_MAX',
            '\xA5':'OP_WITHIN', '\xA6':'OP_RIPEMD160', '\xA7':'OP_SHA1', '\xA8':'OP_SHA256', '\xA9':'OP_HASH160', '\xAA':'OP_HASH256',
            '\xAB':'OP_CODESEPARATOR ', '\xAC':'OP_CHECKSIG', '\xAD':'OP_CHECKSIGVERIFY', '\xAE':'OP_CHECKMULTISIG', '\xAF':'OP_CHECKMULTISIGVERIFY',
            '\xFD':'OP_PUBKEYHASH', '\xFE':'OP_PUBKEY', '\xFF':'OP_INVALIDOPCODE', '\x50':'OP_RESERVED', '\x62':'OP_VER', '\x65':'OP_VERIF',
            '\x66':'OP_VERNOTIF', '\x89':'OP_RESERVED1', '\x8A':'OP_RESERVED2' }

def mkOpCodeStr(data, sepOP=' ', sepPUSH='\n'):
    ops,pos = '',0
    while pos < len(data):
        if data[pos] == 0:
            ops += 'OP_0'+sepOP
        elif data[pos] <= '\x4C':
            sz, = unpack('<B', data[pos])
            ops += data[pos+1:pos+1+sz].encode('hex')+sepPUSH
            pos += sz
        elif data[pos] == '\x4C':
            sz, = unpack('<B', data[pos+1:])
            ops += data[pos+2:pos+2+sz].encode('hex')+sepPUSH
            pos += sz
        elif data[pos] == '\x4D':
            sz, = unpack('<H', data[pos+1:])
            ops += data[pos+2:pos+2+sz].encode('hex')+sepPUSH
            pos += sz
        elif data[pos] == '\x4E':
            sz, = unpack('<I', data[pos+1:])
            ops += data[pos+2:pos+2+sz].encode('hex')+sepPUSH
            pos += sz
        elif data[pos] >= '\x50' and data[pos] <= '\x60':
            ops += 'OP_'+str(int(data[pos]))+sepOP
        elif data[pos] >= '\xB0' and data[pos] <= '\xB9':
            ops += 'OP_NOP'+str(int(data[pos])+1)+sepOP
        else:
            ops += OpCodes[data[pos]]+sepOP
        pos += 1
    return ops

def decodeVarInt(v):
    if v[0] <= '\xfc':
        return unpack('<B', v[0])[0],1
    if v[0] == '\xfd':
        return unpack('<H', v[1:3])[0],3
    if v[0] == '\xfe':
        return unpack('<I', v[1:5])[0],5
    return unpack('<Q', v[1:9])[0],9

def encodeVarInt(v):<commit_msg>bug fixes, update op codes<commit_after>#
# Common sqlchain support utils
#
import os, sys, pwd, time, json, threading, re, glob, urllib2, hashlib

from datetime import datetime
from struct import pack, unpack, unpack_from
from bitcoinrpc.authproxy import AuthServiceProxy, JSONRPCException

from backports.functools_lru_cache import lru_cache # pylint:disable=relative-import
from sqlchain.version import coincfg, ADDR_PREFIX, P2SH_PREFIX, P2SH_CHAR, BECH_HRP, BECH32_FLAG, BECH32_LONG, P2SH_FLAG
from sqlchain.version import S3_BLK_SIZE, BLOB_SPLIT_SIZE

tidylog = threading.Lock()

b58 = "123456789ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz"

class dotdict(dict):
    """dot.notation for more concise globals access on dict"""
    def __getattr__(self, attr):
        return self.get(attr)
    __setattr__= dict.__setitem__
    __delattr__= dict.__delitem__

# address support stuff
def is_address(addr):
    if addr[:2].lower() == coincfg(BECH_HRP):
        return bech32decode(addr) != None
    try:
        n = 0
        for c in addr:
            n = n * 58 + b58.index(c)
        btc = ('%%0%dx' % (25 << 1) % n).decode('hex')[-25:]
        return btc[-4:] == hashlib.sha256(hashlib.sha256(btc[:-4]).digest()).digest()[:4]
    except Exception:
        return False

def mkpkh(pk):
    rmd = hashlib.new('ripemd160')
    rmd.update(hashlib.sha256(pk).digest())
    return rmd.digest()

def addr2pkh(addr):
    if addr[:2].lower() == coincfg(BECH_HRP):
        pkh = bech32decode(addr)
        return str(pkh[2:]) if pkh else None
    long_value = 0L
    for (i, c) in enumerate(addr[::-1]):
        long_value += b58.find(c) * (58**i)
    result = ''
    while long_value >= 256:
        div, mod = divmod(long_value, 256)
        result = chr(mod) + result
        long_value = div
    result = chr(long_value) + result
    nPad = 0
    for c in addr:
        if c == b58[0]:
            nPad += 1
        else:
            break
    result = chr(0)*nPad + result
    return result[1:-4]

def mkaddr(pkh, aid=None, p2sh=False, bech32=False):
    if pkh == '\0'*20 and aid==0:
        return '' # pkh==0 id==0, special case for null address when op_return or non-std script
    if bech32 or (aid and (aid & BECH32_FLAG != 0)):
        return bech32encode(coincfg(BECH_HRP), pkh)
    pad = ''
    an = chr(coincfg(ADDR_PREFIX) if (aid is None and not p2sh) or (aid is not None and (aid & P2SH_FLAG != P2SH_FLAG)) else coincfg(P2SH_PREFIX)) + str(pkh)
    for c in an:
        if c == '\0':
            pad += '1'
        else:
            break
    num = long((an + hashlib.sha256(hashlib.sha256(an).digest()).digest()[0:4]).encode('hex'), 16)
    out = ''
    while num >= 58:
        num,m = divmod(num, 58)
        out = b58[m] + out
    return pad + b58[num] + out

def is_BL32(addr_id):
    return addr_id and (addr_id & BECH32_LONG) == BECH32_LONG

def addr2id(addr, cur=None, rtnPKH=False):
    pkh = addr2pkh(addr)
    if not pkh:
        return None,'' if rtnPKH else None
    addr_id, = unpack('<q', hashlib.sha256(pkh).digest()[:5]+'\0'*3)
    if addr[:2].lower() == coincfg(BECH_HRP): # bech32 has bit 42 set, >20 byte, encode as odd id and stored in bech32 table
        addr_id |= BECH32_FLAG
        if len(pkh) > 20:
            addr_id |= P2SH_FLAG
    elif addr[0] in coincfg(P2SH_CHAR): # encode P2SH flag
        addr_id |= P2SH_FLAG
    if cur:
        tbl = 'bech32' if is_BL32(addr_id) else 'address'
        cur.execute("select id from {0} where id>=%s and id<%s+32 and addr=%s limit 1;".format(tbl), (addr_id,addr_id,pkh))
        row = cur.fetchone()
        return row[0] if row else None
    return (addr_id,pkh) if rtnPKH else addr_id

# script support stuff
def mkSPK(pkh, addr_id):
    if addr_id & BECH32_FLAG: # bech32 id have witness spk
        return ('\x16','\x00\x14%s'%pkh) if (addr_id & P2SH_FLAG != P2SH_FLAG) else ('\x22','\x00\x20%s'%pkh)
    return ('\x19','\x76\xa9\x14%s\x88\xac'%pkh) if (addr_id & P2SH_FLAG != P2SH_FLAG) else ('\x17','\xa9\x14%s\x87'%pkh)

def decodeScriptPK(data):
    result = { 'type':'other', 'data':data } # other, non-std
    if len(data) > 1:
        if len(data) == 25 and data[:3] == '\x76\xa9\x14' and data[23:25] == '\x88\xac': # P2PKH
            result = { 'type':'p2pkh', 'data':'', 'addr':mkaddr(data[3:23]) }
        elif len(data) == 23 and data[:2] == '\xa9\x14' and data[22] == '\x87': # P2SH
            result = { 'type':'p2sh', 'data':'', 'addr':mkaddr(data[2:22],p2sh=True)}
        elif len(data) == 67 and data[0] == '\x41' and data[66] == '\xac': # P2PK
            result = { 'type':'p2pk', 'data':data, 'addr':mkaddr(mkpkh(data[1:66])) }
        elif len(data) >= 35 and data[0] == '\x21' and data[34] == '\xac': # P2PK (compressed key)
            result = { 'type':'p2pk', 'data':data, 'addr':mkaddr(mkpkh(data[1:34])) }
        elif len(data) == 22 and data[:2] == '\x00\x14': # P2WPKH
            result = { 'type':'p2wpkh', 'data':'', 'addr':mkaddr(data[2:22],bech32=True) }
        elif len(data) == 34 and data[:2] == '\x00\x20': # P2WSH
            result = { 'type':'p2wsh', 'data':'', 'addr':mkaddr(data[2:34],bech32=True) }
        elif len(data) >= 38 and data[:6] == '\x6a\x24\xaa\x21\xa9\xed': # witness commitment
            result = { 'type':'witness', 'hash':data[6:38], 'data':data[38:] }
        elif len(data) <= 41 and data[0] == '\x6a': # OP_RETURN
            result = { 'type':'null', 'data':data }
    return result

OpCodes = { '\x4F':'OP_1NEGATE', '\x61':'OP_NOP', '\x63':'OP_IF', '\x64':'OP_NOTIF', '\x67':'OP_ELSE', '\x68':'OP_ENDIF',
            '\x69':'OP_VERIFY', '\x6A':'OP_RETURN', '\x6B':'OP_TOALTSTACK', '\x6C':'OP_FROMALTSTACK', '\x6D':'OP_2DROP', '\x6E':'OP_2DUP',
            '\x6F':'OP_3DUP', '\x70':'OP_2OVER', '\x71':'OP_2ROT', '\x72':'OP_2SWAP','\x73':'OP_IFDUP', '\x74':'OP_DEPTH', '\x75':'OP_DROP',
            '\x76':'OP_DUP', '\x77':'OP_NIP', '\x78':'OP_OVER', '\x79':'OP_PICK', '\x7A':'OP_ROLL', '\x7B':'OP_ROT', '\x7C':'OP_SWAP',
            '\x7D':'OP_TUCK',  '\x7E':'OP_CAT', '\x7F':'OP_SUBSTR', '\x80':'OP_LEFT', '\x81':'OP_RIGHT', '\x82':'OP_SIZE', '\x83':'OP_INVERT',
            '\x84':'OP_AND', '\x85':'OP_OR', '\x86':'OP_XOR', '\x87':'OP_EQUAL', '\x88':'OP_EQUALVERIFY', '\x8B':'OP_1ADD', '\x8C':'OP_1SUB',
            '\x8D':'OP_2MUL', '\x8E':'OP_2DIV', '\x8F':'OP_NEGATE', '\x90':'OP_ABS', '\x91':'OP_NOT', '\x92':'OP_0NOTEQUAL', '\x93':'OP_ADD',
            '\x94':'OP_SUB', '\x95':'OP_MUL', '\x96':'OP_DIV', '\x97':'OP_MOD', '\x98':'OP_LSHIFT', '\x99':'OP_RSHIFT', '\x9A':'OP_BOOLAND',
            '\x9B':'OP_BOOLOR', '\x9C':'OP_NUMEQUAL', '\x9D':'OP_NUMEQUALVERIFY', '\x9E':'OP_NUMNOTEQUAL', '\x9F':'OP_LESSTHAN',
            '\xA0':'OP_GREATERTHAN', '\xA1':'OP_LESSTHANOREQUAL', '\xA2':'OP_GREATERTHANOREQUAL', '\xA3':'OP_MIN', '\xA4':'OP_MAX',
            '\xA5':'OP_WITHIN', '\xA6':'OP_RIPEMD160', '\xA7':'OP_SHA1', '\xA8':'OP_SHA256', '\xA9':'OP_HASH160', '\xAA':'OP_HASH256',
            '\xAB':'OP_CODESEPARATOR ', '\xAC':'OP_CHECKSIG', '\xAD':'OP_CHECKSIGVERIFY', '\xAE':'OP_CHECKMULTISIG', '\xAF':'OP_CHECKMULTISIGVERIFY',
            '\xFD':'OP_PUBKEYHASH', '\xFE':'OP_PUBKEY', '\xFF':'OP_INVALIDOPCODE', '\x50':'OP_RESERVED', '\x62':'OP_VER', '\x65':'OP_VERIF',
            '\x66':'OP_VERNOTIF', '\x89':'OP_RESERVED1', '\x8A':'OP_RESERVED2', '\xB1':'OP_CHECKLOCKTIMEVERIFY', '\xB2':'OP_CHECKSEQUENCEVERIFY' }

def mkOpCodeStr(data, sepOP=' ', sepPUSH='\n'):
    ops,pos = '',0
    while pos < len(data):
        if data[pos] == '\x00':
            ops += 'OP_0'+sepOP
        elif data[pos] <= '\x4C':
            sz, = unpack('<B', data[pos])
            ops += data[pos+1:pos+1+sz].encode('hex')+sepPUSH
            pos += sz
        elif data[pos] == '\x4C':
            sz, = unpack('<B', data[pos+1:])
            ops += data[pos+2:pos+2+sz].encode('hex')+sepPUSH
            pos += sz
        elif data[pos] == '\x4D':
            sz, = unpack('<H', data[pos+1:])
            ops += data[pos+2:pos+2+sz].encode('hex')+sepPUSH
            pos += sz
        elif data[pos] == '\x4E':
            sz, = unpack('<I', data[pos+1:])
            ops += data[pos+2:pos+2+sz].encode('hex')+sepPUSH
            pos += sz
        elif data[pos] >= '\x50' and data[pos] <= '\x60':
            ops += 'OP_'+str(int(ord(data[pos]))-0x50)+sepOP
        elif data[pos] >= '\xB3' and data[pos] <= '\xB9':
            ops += 'OP_NOP'+str(int(ord(data[pos])-0xB2)+1)+sepOP
        else:
            ops += OpCodes[data[pos]]+sepOP
        pos += 1
    return ops[:-1]

def decodeVarInt(v):
    if v[0] <= '\xfc':
        return unpack('<B', v[0])[0],1
    if v[0] == '\xfd':
        return unpack('<H', v[1:3])[0],3
    if v[0] == '\xfe':
        return unpack('<I', v[1:5])[0],5
    return unpack('<Q', v[1:9])[0],9

def encodeVarInt(v):
{'fraction_non_alphanumeric': 0.20770682065062032, 'fraction_numerical': 0.0557429535189526, 'mean_word_length': 3.0211494252873563, 'pattern_counts': {'":': 0, '<': 39, '<?xml version=': 0, '>': 23, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1407874', 'n_tokens_mistral': 7586, 'n_tokens_neox': 6927, 'n_words': 1373}