<commit_before>import re
from random import seed as _seed, choice, randint
from math import pow

VARIABLE_TAG_START = '{{'
VARIABLE_TAG_END = '}}'

tag_re = re.compile('(%s.*?%s)' %
        (re.escape(VARIABLE_TAG_START), re.escape(VARIABLE_TAG_END)))

def choice_words(phrase_book, root_key='root', seed=None):
    if isinstance(seed, int):
        seed = str(seed << 10)
    _seed(seed)
    return eval_phrase(phrase_book, lookup_phrase(root_key, phrase_book))

def eval_phrase(phrase_book, phrase):
    s = ""
    for token in tokenize(phrase):
        if not token.in_tag:
            s += token.text
        else:
            phrase = lookup_phrase(token.text, phrase_book)
            s += eval_phrase(phrase_book, phrase)
    return s

class Token(object):

    def __init__(self, text, in_tag):
        self.in_tag = in_tag
        if in_tag:
            self.text = text[2:-2].strip()
        else:
            self.text = text

def tokenize(phrase):
    result = []
    in_tag = False
    for part in tag_re.split(phrase):
        if part:
            result.append(Token(part, in_tag))
        in_tag = not in_tag
    return result

def lookup_phrase(phrase_key, phrase_book):
    v = phrase_book.get(phrase_key)
    if v is None:
        raise ValueError('Could not find phrase with key "%s".' % phrase_key)
    assert isinstance(v, list)
    return choice(v)

def parse_phrase_book(fname):
    phrases = []
    current_phrase = None 
    for line_number, line in enumerate(open(fname).readlines()):
        line = line.strip()
        if line.startswith('#'):
            # Ignore lines with comments.
            continue
        elif len(line) == 0:
            # Empty lines reset the phrase.
            current_phrase = None
        elif line.startswith('- '):
            # Phrases are prefixed with "- ".
            if current_phrase is None:<commit_msg>Use token type instead of in_tag.<commit_after>import re
from random import seed as _seed, choice, randint
from math import pow

VARIABLE_TAG_START = '{{'
VARIABLE_TAG_END = '}}'

tag_re = re.compile('(%s.*?%s)' %
        (re.escape(VARIABLE_TAG_START), re.escape(VARIABLE_TAG_END)))

def choice_words(phrase_book, root_key='root', seed=None):
    if isinstance(seed, int):
        seed = str(seed << 10)
    _seed(seed)
    return eval_phrase(phrase_book, lookup_phrase(root_key, phrase_book))

def eval_phrase(phrase_book, phrase):
    s = ""
    for token in tokenize(phrase):
        if token.token_type == TOKEN_TEXT:
            s += token.text
        else:
            phrase = lookup_phrase(token.text, phrase_book)
            s += eval_phrase(phrase_book, phrase)
    return s

TOKEN_TEXT = 'text'
TOKEN_REF = 'ref'

class Token(object):

    def __init__(self, text, token_type):
        self.token_type = token_type
        if token_type == TOKEN_REF:
            self.text = text[2:-2].strip()
        else:
            self.text = text

def tokenize(phrase):
    result = []
    in_tag = False
    for part in tag_re.split(phrase):
        if part:
            token_type = TOKEN_REF if in_tag else TOKEN_TEXT
            result.append(Token(part, token_type))
        in_tag = not in_tag
    return result

def lookup_phrase(phrase_key, phrase_book):
    v = phrase_book.get(phrase_key)
    if v is None:
        raise ValueError('Could not find phrase with key "%s".' % phrase_key)
    assert isinstance(v, list)
    return choice(v)

def parse_phrase_book(fname):
    phrases = []
    current_phrase = None 
    for line_number, line in enumerate(open(fname).readlines()):
        line = line.strip()
        if line.startswith('#'):
            # Ignore lines with comments.
            continue
        elif len(line) == 0:
            # Empty lines reset the phrase.
            current_phrase = None
        elif line.startswith('- '):
            # Phrases are prefixed with "- ".
            if current_phrase is None:
{'fraction_non_alphanumeric': 0.1077355118713301, 'fraction_numerical': 0.0025529742149604288, 'mean_word_length': 2.46113074204947, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2075287', 'n_tokens_mistral': 1243, 'n_tokens_neox': 1230, 'n_words': 319}