<commit_before>'''Stranding fee rate calculation

This module contains functions for calculating the stranding fee rate.
'''
from __future__ import division

import multiprocessing
from random import random
from feemodel.config import minrelaytxfee
from feemodel.util import DataSample

__all__ = ['tx_preprocess', 'calc_stranding_feerate']


def tx_preprocess(memblock):
    '''Preprocess MemBlock transactions for calculating stranding fee rate.

    Returns a list of transactions represented by (feerate, inblock), which
    fulfil certain criteria.

    Arguments:
        memblock - A MemBlock object
    '''
    min_leadtime = _calc_min_leadtime(memblock)

    txs = [
        (entry.feerate, entry.inblock)
        for entry in memblock.entries.itervalues()
        if _deps_check(entry, memblock.entries) and
        entry.leadtime >= min_leadtime and
        not entry.isconflict and
        not entry.is_high_priority()]

    return txs


def calc_stranding_feerate(txs, bootstrap=True, multiprocess=None):
    '''Compute stranding feerate from preprocessed txs.

    txs is [(feerate, inblock) for some list of txs].
    bootstrap specifies whether or not to compute error estimates using
    bootstrap resampling.

    multiprocess is the number of processes to use. defaults to
    multiprocessing.cpu_count().
    '''
    if not len(txs):
        raise ValueError('Empty txs list')

    txs.sort(key=lambda x: x[0], reverse=True)
    sfr = _calc_stranding_single(txs)
    abovek = aboven = belowk = belown = 0
    alt_bias_ref = None
    for tx in txs:
        if tx[0] >= sfr:
            abovek += tx[1]
            aboven += 1
        else:
            if alt_bias_ref is None:
                alt_bias_ref = tx[0]
            belowk += not tx[1]
            belown += 1
    if alt_bias_ref is None:
        alt_bias_ref = minrelaytxfee

    if bootstrap and sfr != float("inf"):
        numprocesses = (
            multiprocess if multiprocess is not None
            else multiprocessing.cpu_count())
        workers = multiprocessing.Pool(processes=numprocesses)
        Nchunk = 1000 // numprocesses
        result = workers.map_async(processwork, [(txs, Nchunk)]*numprocesses)
        bs_estimates = sum(result.get(), [])

        if not any([b == float("inf") for b in bs_estimates]):
            datasample = DataSample(bs_estimates)
            datasample.calc_stats()
            mean = datasample.mean
            std = datasample.std
            bias = mean - alt_bias_ref
            alt_bias = mean - sfr
            if abs(alt_bias) > abs(bias):
                bias = alt_bias
        else:
            bias = std = mean = float("inf")
    else:
        bias = std = mean = float("inf")
<commit_msg>stranding doesn't use multiprocess if cpu_count=1<commit_after>'''Stranding fee rate calculation

This module contains functions for calculating the stranding fee rate.
'''
from __future__ import division

import multiprocessing
from random import random
from feemodel.config import minrelaytxfee
from feemodel.util import DataSample

__all__ = ['tx_preprocess', 'calc_stranding_feerate']


def tx_preprocess(memblock):
    '''Preprocess MemBlock transactions for calculating stranding fee rate.

    Returns a list of transactions represented by (feerate, inblock), which
    fulfil certain criteria.

    Arguments:
        memblock - A MemBlock object
    '''
    min_leadtime = _calc_min_leadtime(memblock)

    txs = [
        (entry.feerate, entry.inblock)
        for entry in memblock.entries.itervalues()
        if _deps_check(entry, memblock.entries) and
        entry.leadtime >= min_leadtime and
        not entry.isconflict and
        not entry.is_high_priority()]

    return txs


def calc_stranding_feerate(txs, bootstrap=True, multiprocess=None):
    '''Compute stranding feerate from preprocessed txs.

    txs is [(feerate, inblock) for some list of txs].
    bootstrap specifies whether or not to compute error estimates using
    bootstrap resampling.

    multiprocess is the number of processes to use. defaults to
    multiprocessing.cpu_count().
    '''
    if not len(txs):
        raise ValueError('Empty txs list')

    txs.sort(key=lambda x: x[0], reverse=True)
    sfr = _calc_stranding_single(txs)
    abovek = aboven = belowk = belown = 0
    alt_bias_ref = None
    for tx in txs:
        if tx[0] >= sfr:
            abovek += tx[1]
            aboven += 1
        else:
            if alt_bias_ref is None:
                alt_bias_ref = tx[0]
            belowk += not tx[1]
            belown += 1
    if alt_bias_ref is None:
        alt_bias_ref = minrelaytxfee

    if bootstrap and sfr != float("inf"):
        N = 1000  # Number of bootstrap estimates
        numprocesses = (
            multiprocess if multiprocess is not None
            else multiprocessing.cpu_count())
        if numprocesses == 1:
            bs_estimates = [_calc_stranding_single(bootstrap_sample(txs))
                            for i in range(N)]
        else:
            workers = multiprocessing.Pool(processes=numprocesses)
            Nchunk = N // numprocesses
            result = workers.map_async(
                processwork, [(txs, Nchunk)]*numprocesses)
            bs_estimates = sum(result.get(), [])
            workers.terminate()

        if not any([b == float("inf") for b in bs_estimates]):
            datasample = DataSample(bs_estimates)
            datasample.calc_stats()
            mean = datasample.mean
            std = datasample.std
            bias = mean - alt_bias_ref
            alt_bias = mean - sfr
            if abs(alt_bias) > abs(bias):
                bias = alt_bias
        else:
            bias = std = mean = float("inf")
    else:
        bias = std = mean = float("inf")

{'fraction_non_alphanumeric': 0.08202111091884409, 'fraction_numerical': 0.004499048278248832, 'mean_word_length': 2.3526682134570764, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6679371', 'n_tokens_mistral': 1796, 'n_tokens_neox': 1674, 'n_words': 499}