<commit_before># Copyright (C) 2011 <NAME>

import math
import re


class Scorer:
    def __init__(self, reverse=False):
        self.reverse = reverse

    def finish(self, score):
        if self.reverse:
            score = 1.0 - score

        return score

    def normalize(self, score):
        # map high-valued scores into 0..1
        if score < 0:
            return score

        return 1.0 - 1.0 / (1.0 + score)

    def score(self, input_tokens, output_tokens, db_cache):
        return NotImplementedError


class ScorerGroup:
    def __init__(self):
        self.scorers = []

    def add_scorer(self, weight, scorer):
        self.scorers.append((weight, scorer))

    def score(self, input_tokens, output_tokens, db):
        score = 0.

        for weight, scorer in self.scorers:
            score += weight * scorer.score(input_tokens, output_tokens, db)

        return score


class CobeScorer(Scorer):
    """Classic Cobe scorer, similar to MegaHAL's scorer but with bugs"""
    def __init__(self, order, **kwargs):
        Scorer.__init__(self, **kwargs)
        self.order = order

    def score(self, input_tokens, output_tokens, db):
        # If input_tokens is empty (i.e. we didn't know any words in
        # the input), use output == input to make sure we still check
        # scoring
        if len(input_tokens) == 0:
            input_tokens = output_tokens

        score = 0.

        # evaluate forward probabilities
        for output_idx in xrange(len(output_tokens) - self.order):
            output_token = output_tokens[output_idx + self.order]
            if output_token in input_tokens:
                expr = output_tokens[output_idx:output_idx + self.order]

                p = db.get_expr_token_probability("next_token", expr,
                                                  output_token)

                if p > 0:
                    score -= math.log(p, 2)

        # evaluate reverse probabilities
        for output_idx in xrange(len(output_tokens) - self.order):
            output_token = output_tokens[output_idx]
            if output_token in input_tokens:
                start = output_idx + 1
                end = start + self.order

                expr = output_tokens[start:end]

                p = db.get_expr_token_probability("prev_token", expr,
                                                  output_token)

                if p > 0:
                    score -= math.log(p, 2)

        raw_score = score

        # Prefer smaller replies. This behavior is present but not
        # documented in recent MegaHAL.
        score_divider = 1
        n_tokens = len(output_tokens)
        if n_tokens >= 8:
            score_divider = math.sqrt(n_tokens - 1)
        elif n_tokens >= 16:
            score_divider = n_tokens

        score = score / score_divider

        return self.finish(self.normalize(score))<commit_msg>Add LengthScorer and ShoutyScorer<commit_after># Copyright (C) 2011 <NAME>

import math
import re


class Scorer:
    def __init__(self, reverse=False):
        self.reverse = reverse

    def finish(self, score):
        if self.reverse:
            score = 1.0 - score

        return score

    def normalize(self, score):
        # map high-valued scores into 0..1
        if score < 0:
            return score

        return 1.0 - 1.0 / (1.0 + score)

    def score(self, input_tokens, output_tokens, db_cache):
        return NotImplementedError


class ScorerGroup:
    def __init__(self):
        self.scorers = []

    def add_scorer(self, weight, scorer):
        self.scorers.append((weight, scorer))

    def score(self, input_tokens, output_tokens, db):
        score = 0.

        for weight, scorer in self.scorers:
            score += weight * scorer.score(input_tokens, output_tokens, db)

        return score


class CobeScorer(Scorer):
    """Classic Cobe scorer, similar to MegaHAL's scorer but with bugs"""
    def __init__(self, order, **kwargs):
        Scorer.__init__(self, **kwargs)
        self.order = order

    def score(self, input_tokens, output_tokens, db):
        # If input_tokens is empty (i.e. we didn't know any words in
        # the input), use output == input to make sure we still check
        # scoring
        if len(input_tokens) == 0:
            input_tokens = output_tokens

        score = 0.

        # evaluate forward probabilities
        for output_idx in xrange(len(output_tokens) - self.order):
            output_token = output_tokens[output_idx + self.order]
            if output_token in input_tokens:
                expr = output_tokens[output_idx:output_idx + self.order]

                p = db.get_expr_token_probability("next_token", expr,
                                                  output_token)

                if p > 0:
                    score -= math.log(p, 2)

        # evaluate reverse probabilities
        for output_idx in xrange(len(output_tokens) - self.order):
            output_token = output_tokens[output_idx]
            if output_token in input_tokens:
                start = output_idx + 1
                end = start + self.order

                expr = output_tokens[start:end]

                p = db.get_expr_token_probability("prev_token", expr,
                                                  output_token)

                if p > 0:
                    score -= math.log(p, 2)

        raw_score = score

        # Prefer smaller replies. This behavior is present but not
        # documented in recent MegaHAL.
        score_divider = 1
        n_tokens = len(output_tokens)
        if n_tokens >= 8:
            score_divider = math.sqrt(n_tokens - 1)
        elif n_tokens >= 16:
            score_divider = n_tokens

        score = score / score_divider

        return self.finish(self.normalize(score))


class LengthScorer(Scorer):
    def score(self, input_tokens, output_tokens, db):
        return self.finish(self.normalize(len(output_tokens)))


class ShoutyScorer(Scorer):
    def score(self, input_tokens, output_tokens, db):
        infos = [db.get_token_info(token_id) for token_id in output_tokens]

        words = []
        for info in infos:
            if info["is_word"] and len(info["text"]) > 1:
                words.append(info["text"])

        shouty_count = 0
        for word in words:
            if word == word.upper():
                shouty_count += 1

        return self.finish(float(shouty_count) / len(words))
{'fraction_non_alphanumeric': 0.08587343339006653, 'fraction_numerical': 0.009128887513538604, 'mean_word_length': 1.8463232056362835, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 14, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2705488', 'n_tokens_mistral': 1919, 'n_tokens_neox': 1772, 'n_words': 511}