<commit_before>from stochastics.stochastic_with_log_p import StochasticWithLogP
import numpy as np

class TalentPoolValueAccessor(object):

    @classmethod
    def get_mean_skill_mean(self, value):
        return value[0]

    @classmethod
    def get_mean_skill_std_dev(self, value):
        return value[1]

    @classmethod
    def get_log_of_std_dev_of_skill_mean(self, value):
        return value[2]

    @classmethod
    def get_log_of_std_dev_of_skill_std_dev(self, value):
        return value[3]

    @classmethod
    def build_value(cls,
                    mean_skill_mean,
                    mean_skill_std_dev,
                    log_of_std_dev_of_mean_skill_mean,
                    log_of_std_dev_of_skill_std_dev):

        return np.array([mean_skill_mean,
                         mean_skill_std_dev,
                         log_of_std_dev_of_mean_skill_mean,
                         log_of_std_dev_of_skill_std_dev], dtype=float)

SQRT_TWO_PI = (2*np.pi)**.5

def compute_gaussian_logp(x, mu, sigma):
    exp_arg = -(x-mu)**2 / (2*sigma**2)
    norm_term = -np.log(sigma*SQRT_TWO_PI)
    return exp_arg + norm_term

def draw_random_gaussian(mu, sigma):
    return mu + np.random.randn()*sigma


class TalentPool(StochasticWithLogP):


    PRIOR_MEAN = 0.0

    @classmethod
    def _compute_logp(cls, value, **parent_values):

        mean_skill_mean = TalentPoolValueAccessor.get_mean_skill_mean(value)
        mean_skill_std_dev = TalentPoolValueAccessor.get_mean_skill_std_dev(value)

        #std_dev of skill doesn't come into play, meaning all variances are a-priori equally likely
        return compute_gaussian_logp(mean_skill_mean, cls.PRIOR_MEAN, mean_skill_std_dev)<commit_msg>BF: fix computation of log p for talent pool<commit_after>from stochastics.stochastic_with_log_p import StochasticWithLogP
import numpy as np

class TalentPoolValueAccessor(object):

    @classmethod
    def get_mean_skill_mean(cls, value):
        return value[0]

    @classmethod
    def get_mean_skill_std_dev(cls, value):
        return value[1]

    @classmethod
    def get_log_of_std_dev_of_skill_mean(cls, value):
        return value[2]

    @classmethod
    def get_log_of_std_dev_of_skill_std_dev(cls, value):
        return value[3]

    @classmethod
    def build_value(cls,
                    mean_skill_mean,
                    mean_skill_std_dev,
                    log_of_std_dev_of_skill_mean,
                    log_of_std_dev_of_skill_std_dev):

        return np.array([mean_skill_mean,
                         mean_skill_std_dev,
                         log_of_std_dev_of_skill_mean,
                         log_of_std_dev_of_skill_std_dev], dtype=float)

SQRT_TWO_PI = (2*np.pi)**.5

def compute_gaussian_logp(x, mu, sigma):
    exp_arg = -(x-mu)**2 / (2*sigma**2)
    norm_term = -np.log(sigma*SQRT_TWO_PI)
    return exp_arg + norm_term

def draw_random_gaussian(mu, sigma):
    return mu + np.random.randn()*sigma


class TalentPool(StochasticWithLogP):


    PRIOR_MEAN = 0.0
    PRIOR_STD_DEV = 1.0

    @classmethod
    def _compute_logp(cls, value, **parent_values):

        # the mean skill is anchored, but we have flat priors over:
        # 1. the std dev of the mean skill. we don't know how by much variation in mean skill there is between individuals.
        # 2. the mean of the std dev of the skill. we don't know how variable an individual's performance is likely to be.
        # 3. the std dev of the std dev of the skill. we don't know how wild is the wildest individual or how consistent the most consistent
        mean_skill_mean = TalentPoolValueAccessor.get_mean_skill_mean(value)
        logp = compute_gaussian_logp(mean_skill_mean, cls.PRIOR_MEAN, cls.PRIOR_STD_DEV)

        return logp
{'fraction_non_alphanumeric': 0.07095402604305076, 'fraction_numerical': 0.007175126229072549, 'mean_word_length': 2.8135764944275583, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5752804', 'n_tokens_mistral': 1306, 'n_tokens_neox': 1265, 'n_words': 272}