<commit_before>#!/usr/bin/env python3
#
# linearize-hashes.py:  List blocks in a linear, no-fork version of the chain.
#
# Copyright (c) 2013-2016 The Bitcoin Core developers
# Distributed under the MIT software license, see the accompanying
# file COPYING or http://www.opensource.org/licenses/mit-license.php.
#

from __future__ import print_function
try:  # Python 3
    import http.client as httplib
except ImportError:  # Python 2
    import httplib
import json
import re
import base64
import sys

settings = {}

##### Switch endian-ness #####


def hex_switchEndian(s):
    """ Switches the endianness of a hex string (in pairs of hex chars) """
    pairList = [s[i:i + 2].encode() for i in range(0, len(s), 2)]
    return b''.join(pairList[::-1]).decode()


class BitcoinRPC:
    def __init__(self, host, port, username, password):
        authpair = "%s:%s" % (username, password)
        authpair = authpair.encode('utf-8')
        self.authhdr = b"Basic " + base64.b64encode(authpair)
        self.conn = httplib.HTTPConnection(host, port=port, timeout=30)

    def execute(self, obj):
        try:
            self.conn.request('POST', '/', json.dumps(obj),
                              {'Authorization': self.authhdr,
                               'Content-type': 'application/json'})
        except ConnectionRefusedError:
            print('RPC connection refused. Check RPC settings and the server status.',
                  file=sys.stderr)
            return None

        resp = self.conn.getresponse()
        if resp is None:
            print("JSON-RPC: no response", file=sys.stderr)
            return None

        body = resp.read().decode('utf-8')
        resp_obj = json.loads(body)
        return resp_obj

    @staticmethod
    def build_request(idx, method, params):
        obj = {'version': '1.1',
               'method': method,
               'id': idx}
        if params is None:
            obj['params'] = []
        else:
            obj['params'] = params
        return obj

    @staticmethod
    def response_is_error(resp_obj):
        return 'error' in resp_obj and resp_obj['error'] is not None


def get_block_hashes(settings, max_blocks_per_call=10000):
    rpc = BitcoinRPC(settings['host'], settings['port'],
                     settings['rpcuser'], settings['rpcpassword'])

    height = settings['min_height']
    while height < settings['max_height'] + 1:
        num_blocks = min(settings['max_height'] +
                         1 - height, max_blocks_per_call)
        batch = []
        for x in range(num_blocks):
            batch.append(rpc.build_request(x, 'getblockhash', [height + x]))

        reply = rpc.execute(batch)
        if reply is None:
            print('Cannot continue. Program will halt.')
            return None

        for x, resp_obj in enumerate(reply):
            if rpc.response_is_error(resp_obj):
                print('JSON-RPC: error at height', height + x,
                      ': ', resp_obj['error'], file=sys.stderr)
                exit(1)
            assert(resp_obj['id'] == x)  # assume replies are in-sequence
            if settings['rev_hash_bytes'] == 'true':
                resp_obj['result'] = hex_switchEndian(resp_obj['result'])
            print(resp_obj['result'])

        height += num_blocks


if __name__ == '__main__':
    if len(sys.argv) != 2:
        print("Usage: linearize-hashes.py CONFIG-FILE")
        sys.exit(1)

    f = open(sys.argv[1])
    for line in f:
        # skip comment lines
        m = re.search('^\s*#', line)
        if m:
            continue

        # parse key=value lines
        m = re.search('^(\w+)\s*=\s*(\S.*)$', line)
        if m is None:
            continue
        settings[m.group(1)] = m.group(2)
    f.close()

    if 'host' not in settings:
        settings['host'] = '1172.16.58.3'
    if 'port' not in settings:
        settings['port'] = 8332
    if 'min_height' not in settings:
        settings['min_height'] = 0
    if 'max_height' not in settings:
        settings['max_height'] = 313000
    if 'rev_hash_bytes' not in settings:
        settings['rev_hash_bytes'] = 'false'
    if 'rpcuser' not in settings or 'rpcpassword' not in settings:
        print("Missing username and/or password in cfg file", file=stderr)
        sys.exit(1)

    settings['port'] = int(settings['port'])
    settings['min_height'] = int(settings['min_height'])
    settings['max_height'] = int(settings['max_height'])

    # Force hash byte format setting to be lowercase to make comparisons easier.
    settings['rev_hash_bytes'] = settings['rev_hash_bytes'].lower()

    get_block_hashes(settings)<commit_msg>Avoid reference to undefined name: stderr does not exist, sys.stderr does<commit_after>#!/usr/bin/env python3
#
# linearize-hashes.py:  List blocks in a linear, no-fork version of the chain.
#
# Copyright (c) 2013-2016 The Bitcoin Core developers
# Distributed under the MIT software license, see the accompanying
# file COPYING or http://www.opensource.org/licenses/mit-license.php.
#

from __future__ import print_function
try:  # Python 3
    import http.client as httplib
except ImportError:  # Python 2
    import httplib
import json
import re
import base64
import sys

settings = {}

##### Switch endian-ness #####


def hex_switchEndian(s):
    """ Switches the endianness of a hex string (in pairs of hex chars) """
    pairList = [s[i:i + 2].encode() for i in range(0, len(s), 2)]
    return b''.join(pairList[::-1]).decode()


class BitcoinRPC:
    def __init__(self, host, port, username, password):
        authpair = "%s:%s" % (username, password)
        authpair = authpair.encode('utf-8')
        self.authhdr = b"Basic " + base64.b64encode(authpair)
        self.conn = httplib.HTTPConnection(host, port=port, timeout=30)

    def execute(self, obj):
        try:
            self.conn.request('POST', '/', json.dumps(obj),
                              {'Authorization': self.authhdr,
                               'Content-type': 'application/json'})
        except ConnectionRefusedError:
            print('RPC connection refused. Check RPC settings and the server status.',
                  file=sys.stderr)
            return None

        resp = self.conn.getresponse()
        if resp is None:
            print("JSON-RPC: no response", file=sys.stderr)
            return None

        body = resp.read().decode('utf-8')
        resp_obj = json.loads(body)
        return resp_obj

    @staticmethod
    def build_request(idx, method, params):
        obj = {'version': '1.1',
               'method': method,
               'id': idx}
        if params is None:
            obj['params'] = []
        else:
            obj['params'] = params
        return obj

    @staticmethod
    def response_is_error(resp_obj):
        return 'error' in resp_obj and resp_obj['error'] is not None


def get_block_hashes(settings, max_blocks_per_call=10000):
    rpc = BitcoinRPC(settings['host'], settings['port'],
                     settings['rpcuser'], settings['rpcpassword'])

    height = settings['min_height']
    while height < settings['max_height'] + 1:
        num_blocks = min(settings['max_height'] +
                         1 - height, max_blocks_per_call)
        batch = []
        for x in range(num_blocks):
            batch.append(rpc.build_request(x, 'getblockhash', [height + x]))

        reply = rpc.execute(batch)
        if reply is None:
            print('Cannot continue. Program will halt.')
            return None

        for x, resp_obj in enumerate(reply):
            if rpc.response_is_error(resp_obj):
                print('JSON-RPC: error at height', height + x,
                      ': ', resp_obj['error'], file=sys.stderr)
                exit(1)
            assert(resp_obj['id'] == x)  # assume replies are in-sequence
            if settings['rev_hash_bytes'] == 'true':
                resp_obj['result'] = hex_switchEndian(resp_obj['result'])
            print(resp_obj['result'])

        height += num_blocks


if __name__ == '__main__':
    if len(sys.argv) != 2:
        print("Usage: linearize-hashes.py CONFIG-FILE")
        sys.exit(1)

    f = open(sys.argv[1])
    for line in f:
        # skip comment lines
        m = re.search('^\s*#', line)
        if m:
            continue

        # parse key=value lines
        m = re.search('^(\w+)\s*=\s*(\S.*)$', line)
        if m is None:
            continue
        settings[m.group(1)] = m.group(2)
    f.close()

    if 'host' not in settings:
        settings['host'] = '127.0.0.1'
    if 'port' not in settings:
        settings['port'] = 8332
    if 'min_height' not in settings:
        settings['min_height'] = 0
    if 'max_height' not in settings:
        settings['max_height'] = 313000
    if 'rev_hash_bytes' not in settings:
        settings['rev_hash_bytes'] = 'false'
    if 'rpcuser' not in settings or 'rpcpassword' not in settings:
        print("Missing username and/or password in cfg file", file=sys.stderr)
        sys.exit(1)

    settings['port'] = int(settings['port'])
    settings['min_height'] = int(settings['min_height'])
    settings['max_height'] = int(settings['max_height'])

    # Force hash byte format setting to be lowercase to make comparisons easier.
    settings['rev_hash_bytes'] = settings['rev_hash_bytes'].lower()

    get_block_hashes(settings)
{'fraction_non_alphanumeric': 0.12709244055869495, 'fraction_numerical': 0.012687919820876426, 'mean_word_length': 2.339266642933428, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 2, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1350575', 'n_tokens_mistral': 2872, 'n_tokens_neox': 2742, 'n_words': 747}