<commit_before>"""
Try to produce manglings like "Tweeze denied beef worker isthmus".
"""
from memo import memo
from pdist import Pw
from math import log10

roughener = {
    'd': 't', 'dh': 't', 'th': 't',
    'l': 'r',
    'sh': 's', 'z': 's', 'zh': 's',
}

def roughen(phones):
    return tuple(p[-1] if p[-1].isdigit() else roughener.get(p, p)
                 for p in phones)

phones_of_word = {}
words_of_phones = {}
rough_words = {}
for line in open('cmudict.0.7a'):
    if ';;' in line: continue
    s = line.lower().split()
    if not s: continue
    word, phones = s[0], tuple(s[1:])
    if word.endswith(')'): continue
    phones_of_word[word] = phones
    words_of_phones.setdefault(phones, []).append(word)
    rough_words.setdefault(roughen(phones), []).append(word)

def pronounce(word):
    if word not in phones_of_word:
        phones_of_word[word] = (word,)
        words_of_phones[word,] = [word]
    return phones_of_word[word]

def pronounce_all(words):
    return sum(map(pronounce, words), ())

def annotated_pronounce_all(words):
    phones, bounds = (), ()
    for word in words:
        i = len(phones)
        p = pronounce(word)
        phones += p
        bounds += (word,) + (None,)*(len(p)-1)
    return phones, bounds

## pronounce('yay')
#. ('y', 'ey1')
## pronounce_all(('darius',))
#. ('d', 'er0', 'ay1', 'ah0', 's')
## annotated_pronounce_all(('darius','is','great'))
#. (('d', 'er0', 'ay1', 'ah0', 's', 'ih1', 'z', 'g', 'r', 'ey1', 't'), ('darius', None, None, None, None, 'is', None, 'great', None, None, None))

## transcribe(*annotated_pronounce_all(tuple("stirring not even a mouse".split())))
#. (21.741929954664087, ('staying', 'note', 'ave', 'mass'))
## transcribe(*annotated_pronounce_all(tuple("snug in their beds with visions".split())))
#. (43.36161186892771, ('snag', "'n", 'there', 'bids', 'wythe', 'versions'))
## transcribe(*annotated_pronounce_all(tuple("syne".split())))
#. (56.71069362546324, ('syne',))

longest = 20
match_cost = 25
fit_cost = 4
rarity_cost = 1
roughened_cost = 1

## transcribe(*annotated_pronounce_all(tuple("the light".split())))
#. (56.50999260901199, ('the', 'let'))

## pronounce_all(tuple("the light".split()))
#. ('dh', 'ah0', 'l', 'ay1', 't')

def compute_best(phones, bounds, costs, seqs, i):
    attempts = []
    for L in range(1, min(i, longest) + 1):
        assert len(phones[:i-L]) < len(phones)
        subcost, subwords = costs[i-L], seqs[i-L]
        subcost += fit_cost*(None is not bounds[i-L])
        def add(word, common_cost):
            attempts.append((common_cost - rarity_cost*log10(Pw(word)) + match_cost*(word == bounds[i-L]),<commit_msg>tweak costs<commit_after>"""
Try to produce manglings like "Tweeze denied beef worker isthmus".
"""
from memo import memo
from pdist import Pw
from math import log10

roughener = {
    'd': 't', 'dh': 't', 'th': 't',
    'l': 'r',
    'sh': 's', 'z': 's', 'zh': 's',
}

def roughen(phones):
    return tuple(p[-1] if p[-1].isdigit() else roughener.get(p, p)
                 for p in phones)

phones_of_word = {}
words_of_phones = {}
rough_words = {}
for line in open('cmudict.0.7a'):
    if ';;' in line: continue
    s = line.lower().split()
    if not s: continue
    word, phones = s[0], tuple(s[1:])
    if word.endswith(')'): continue
    phones_of_word[word] = phones
    words_of_phones.setdefault(phones, []).append(word)
    rough_words.setdefault(roughen(phones), []).append(word)

def pronounce(word):
    if word not in phones_of_word:
        phones_of_word[word] = (word,)
        words_of_phones[word,] = [word]
    return phones_of_word[word]

def pronounce_all(words):
    return sum(map(pronounce, words), ())

def annotated_pronounce_all(words):
    phones, bounds = (), ()
    for word in words:
        i = len(phones)
        p = pronounce(word)
        phones += p
        bounds += (word,) + (None,)*(len(p)-1)
    return phones, bounds

## pronounce('yay')
#. ('y', 'ey1')
## pronounce_all(('darius',))
#. ('d', 'er0', 'ay1', 'ah0', 's')
## annotated_pronounce_all(('darius','is','great'))
#. (('d', 'er0', 'ay1', 'ah0', 's', 'ih1', 'z', 'g', 'r', 'ey1', 't'), ('darius', None, None, None, None, 'is', None, 'great', None, None, None))

## transcribe(*annotated_pronounce_all(tuple("stirring not even a mouse".split())))
#. (21.741929954664087, ('staying', 'note', 'ave', 'mass'))
## transcribe(*annotated_pronounce_all(tuple("snug in their beds with visions".split())))
#. (43.36161186892771, ('snag', "'n", 'there', 'bids', 'wythe', 'versions'))
## transcribe(*annotated_pronounce_all(tuple("syne".split())))
#. (56.71069362546324, ('syne',))

longest = 20
match_cost = 25
fit_cost = 5
rarity_cost = 5
roughened_cost = 5

## transcribe(*annotated_pronounce_all(tuple("the light".split())))
#. (56.50999260901199, ('the', 'let'))

## pronounce_all(tuple("the light".split()))
#. ('dh', 'ah0', 'l', 'ay1', 't')

def compute_best(phones, bounds, costs, seqs, i):
    attempts = []
    for L in range(1, min(i, longest) + 1):
        assert len(phones[:i-L]) < len(phones)
        subcost, subwords = costs[i-L], seqs[i-L]
        subcost += fit_cost*(None is not bounds[i-L])
        def add(word, common_cost):
            attempts.append((common_cost - rarity_cost*log10(Pw(word)) + match_cost*(word == bounds[i-L]),
{'fraction_non_alphanumeric': 0.20688350983358547, 'fraction_numerical': 0.036308623298033284, 'mean_word_length': 4.018026565464895, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5465099', 'n_tokens_mistral': 2129, 'n_tokens_neox': 1984, 'n_words': 464}