<commit_before>from Queue import Queue, Empty
from threading import Thread
import copy
import datetime
import functools
import time
import json
import os
import re
import subprocess
import hashlib

from populus import utils


is_geth_available = functools.partial(utils.is_executable_available, 'geth')
is_nice_available = functools.partial(utils.is_executable_available, 'nice')


POPULUS_DIR = os.path.abspath(os.path.dirname(__file__))
KNOWN_CONTRACTS_FILE = "known_contracts.json"
ACTIVE_CHAIN_SYMLINK = ".active-chain"


def get_blockchains_dir(project_dir):
    blockchains_dir = os.path.abspath(os.path.join(project_dir, 'chains'))
    utils.ensure_path_exists(blockchains_dir)
    return blockchains_dir


def get_geth_data_dir(project_dir, name):
    blockchains_dir = get_blockchains_dir(project_dir)
    data_dir = os.path.abspath(os.path.join(blockchains_dir, name))
    utils.ensure_path_exists(data_dir)
    return data_dir


def get_active_data_dir(project_dir):
    """ Given a project directory, this function creates the
        file path string for the active chain symlink.
    """
    bc_dir = get_blockchains_dir(project_dir)
    active_dir = os.path.abspath(os.path.join(bc_dir, ACTIVE_CHAIN_SYMLINK))
    return active_dir


def set_active_data_dir(project_dir, name):
    """ Set which ethereum chain directory is the
        active chain. This basically just creates a new
        symlink pointed at the active chain.
    """
    data_dir = get_geth_data_dir(project_dir, name)
    active_dir = get_active_data_dir(project_dir)
    if os.path.islink(active_dir):
        os.unlink(active_dir)
    os.symlink(data_dir, active_dir)


def get_geth_logfile_path(data_dir, logfile_name_fmt="geth-{0}.log"):
    logfile_name = logfile_name_fmt.format(
        datetime.datetime.now().strftime("%Y%m%d-%H%M%S"),
    )
    log_dir = os.path.join(data_dir, 'logs')
    utils.ensure_path_exists(log_dir)
    logfile_path = os.path.abspath(os.path.join(log_dir, logfile_name))
    return logfile_path


def enqueue_stream(stream, queue, logfile=None):
    """
    Synchronously reads the output from a stream (stdout, stderr) and puts it
    onto the queue to be read.
    """
    for line in iter(stream.readline, b''):
        queue.put(line)
        if logfile:
            logfile.write(line)
            logfile.flush()
    if logfile:
        try:
            logfile.close()
        except ValueError:
            pass


class PopenWrapper(subprocess.Popen):
    _locked = False

    def __init__(self, args, bufsize='1', stdin=subprocess.PIPE,
                 stdout=subprocess.PIPE, stderr=subprocess.PIPE, **kwargs):
        super(PopenWrapper, self).__init__(args, bufsize=bufsize, stdin=stdin,
                                           stdout=stdout, stderr=stderr,
                                           **kwargs)
        if "--logfile" in args:
            logfile_path = args[args.index("--logfile") + 1]
            logfile = open(logfile_path, 'a')
        else:
            logfile = None
        self.stdout_queue = Queue()
        self.stdout_thread = Thread(
            target=enqueue_stream,
            args=(self.stdout, self.stdout_queue, logfile),
        )
        self.stdout_thread.daemon = True
        self.stdout_thread.start()

        self.stderr_queue = Queue()
        self.stderr_thread = Thread(
            target=enqueue_stream,
            args=(self.stderr, self.stderr_queue, logfile),
        )
        self.stderr_thread.daemon = True
        self.stderr_thread.start()

    def get_stdout_nowait(self):
        try:
            return self.stdout_queue.get_nowait()
        except Empty:
            return None

    def get_stderr_nowait(self):
        try:
            return self.stderr_queue.get_nowait()
        except Empty:
            return None

    _output_generator = None

    def get_output_nowait(self):
        if self._output_generator is None:
            def output_generator():
                while True:
                    yield self.get_stdout_nowait()
                    yield self.get_stderr_nowait()
            self._output_generator = output_generator()
        return self._output_generator.next()

    def communicate(self, *args, **kwargs):
        raise ValueError("Cannot communicate with a PopenWrapper")


DEFAULT_PW_PATH = os.path.join(POPULUS_DIR, 'default_blockchain_password')


def geth_wrapper(data_dir, popen_class=subprocess.Popen, cmd="geth",
                 genesis_block=None, miner_threads='1', extra_args=None,
                 max_peers='0', network_id='123456', no_discover=True,
                 mine=False, nice=True, unlock='0', password=<PASSWORD>,
                 port=None, verbosity=None, logfile=None):
    if nice and is_nice_available():
        command = ['nice', '-n', '20', cmd]
    else:
        command = [cmd]

    if genesis_block is None:
        genesis_block = get_genesis_block_path(data_dir)
        if not os.path.exists(genesis_block):
            genesis_block = os.path.join(POPULUS_DIR, 'genesis-test.json')

    if port is None:
        port = utils.get_open_port()

    command.extend((
        '--genesis', genesis_block,
        '--datadir', data_dir,
        '--maxpeers', max_peers,
        '--networkid', network_id,
        '--port', port,
    ))

    if miner_threads is not None:
        command.extend(('--minerthreads', miner_threads))

    if logfile is not None:
        command.extend((
            '--logfile', logfile,
        ))

    if verbosity is not None:
        command.extend((
            '--verbosity', verbosity,
        ))

    if unlock is not None:<commit_msg>pass an explicit (short) ipcpath to geth because unix socket paths need to be short<commit_after>from Queue import Queue, Empty
from threading import Thread
import copy
import datetime
import functools
import time
import json
import os
import re
import subprocess
import hashlib
import tempfile

from populus import utils


is_geth_available = functools.partial(utils.is_executable_available, 'geth')
is_nice_available = functools.partial(utils.is_executable_available, 'nice')


POPULUS_DIR = os.path.abspath(os.path.dirname(__file__))
KNOWN_CONTRACTS_FILE = "known_contracts.json"
ACTIVE_CHAIN_SYMLINK = ".active-chain"


def get_blockchains_dir(project_dir):
    blockchains_dir = os.path.abspath(os.path.join(project_dir, 'chains'))
    utils.ensure_path_exists(blockchains_dir)
    return blockchains_dir


def get_geth_data_dir(project_dir, name):
    blockchains_dir = get_blockchains_dir(project_dir)
    data_dir = os.path.abspath(os.path.join(blockchains_dir, name))
    utils.ensure_path_exists(data_dir)
    return data_dir


def get_active_data_dir(project_dir):
    """ Given a project directory, this function creates the
        file path string for the active chain symlink.
    """
    bc_dir = get_blockchains_dir(project_dir)
    active_dir = os.path.abspath(os.path.join(bc_dir, ACTIVE_CHAIN_SYMLINK))
    return active_dir


def set_active_data_dir(project_dir, name):
    """ Set which ethereum chain directory is the
        active chain. This basically just creates a new
        symlink pointed at the active chain.
    """
    data_dir = get_geth_data_dir(project_dir, name)
    active_dir = get_active_data_dir(project_dir)
    if os.path.islink(active_dir):
        os.unlink(active_dir)
    os.symlink(data_dir, active_dir)


def get_geth_logfile_path(data_dir, logfile_name_fmt="geth-{0}.log"):
    logfile_name = logfile_name_fmt.format(
        datetime.datetime.now().strftime("%Y%m%d-%H%M%S"),
    )
    log_dir = os.path.join(data_dir, 'logs')
    utils.ensure_path_exists(log_dir)
    logfile_path = os.path.abspath(os.path.join(log_dir, logfile_name))
    return logfile_path


def enqueue_stream(stream, queue, logfile=None):
    """
    Synchronously reads the output from a stream (stdout, stderr) and puts it
    onto the queue to be read.
    """
    for line in iter(stream.readline, b''):
        queue.put(line)
        if logfile:
            logfile.write(line)
            logfile.flush()
    if logfile:
        try:
            logfile.close()
        except ValueError:
            pass


class PopenWrapper(subprocess.Popen):
    _locked = False

    def __init__(self, args, bufsize='1', stdin=subprocess.PIPE,
                 stdout=subprocess.PIPE, stderr=subprocess.PIPE, **kwargs):
        super(PopenWrapper, self).__init__(args, bufsize=bufsize, stdin=stdin,
                                           stdout=stdout, stderr=stderr,
                                           **kwargs)
        if "--logfile" in args:
            logfile_path = args[args.index("--logfile") + 1]
            logfile = open(logfile_path, 'a')
        else:
            logfile = None
        self.stdout_queue = Queue()
        self.stdout_thread = Thread(
            target=enqueue_stream,
            args=(self.stdout, self.stdout_queue, logfile),
        )
        self.stdout_thread.daemon = True
        self.stdout_thread.start()

        self.stderr_queue = Queue()
        self.stderr_thread = Thread(
            target=enqueue_stream,
            args=(self.stderr, self.stderr_queue, logfile),
        )
        self.stderr_thread.daemon = True
        self.stderr_thread.start()

    def get_stdout_nowait(self):
        try:
            return self.stdout_queue.get_nowait()
        except Empty:
            return None

    def get_stderr_nowait(self):
        try:
            return self.stderr_queue.get_nowait()
        except Empty:
            return None

    _output_generator = None

    def get_output_nowait(self):
        if self._output_generator is None:
            def output_generator():
                while True:
                    yield self.get_stdout_nowait()
                    yield self.get_stderr_nowait()
            self._output_generator = output_generator()
        return self._output_generator.next()

    def communicate(self, *args, **kwargs):
        raise ValueError("Cannot communicate with a PopenWrapper")


DEFAULT_PW_PATH = os.path.join(POPULUS_DIR, 'default_blockchain_password')


def geth_wrapper(data_dir, popen_class=subprocess.Popen, cmd="geth",
                 genesis_block=None, miner_threads='1', extra_args=None,
                 max_peers='0', network_id='123456', no_discover=True,
                 mine=False, nice=True, unlock='0', password=<PASSWORD>,
                 port=None, verbosity=None, logfile=None, ipcpath=None):
    if nice and is_nice_available():
        command = ['nice', '-n', '20', cmd]
    else:
        command = [cmd]

    if genesis_block is None:
        genesis_block = get_genesis_block_path(data_dir)
        if not os.path.exists(genesis_block):
            genesis_block = os.path.join(POPULUS_DIR, 'genesis-test.json')

    if port is None:
        port = utils.get_open_port()
    
    if ipcpath is None:
        ipcpath = tempfile.NamedTemporaryFile().name

    command.extend((
        '--genesis', genesis_block,
        '--datadir', data_dir,
        '--maxpeers', max_peers,
        '--networkid', network_id,
        '--port', port,
        '--ipcpath', ipcpath,
    ))

    if miner_threads is not None:
        command.extend(('--minerthreads', miner_threads))

    if logfile is not None:
        command.extend((
            '--logfile', logfile,
        ))

    if verbosity is not None:
        command.extend((
            '--verbosity', verbosity,
        ))

    if unlock is not None:
{'fraction_non_alphanumeric': 0.0991994430908458, 'fraction_numerical': 0.0024364775495997215, 'mean_word_length': 2.587078651685393, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1626035', 'n_tokens_mistral': 3643, 'n_tokens_neox': 3544, 'n_words': 756}