# Path: languages/utils.py
# def word_triples(word):
#     """
#     >>> list(word_triples('weight'))
#     ['^wei', 'weight', 'eight$']
#     >>> list(word_triples('eightyfive'))
#     ['^eight', 'eighty', 'ghtyf', 'yfi', 'fiv', 'ive', 've$']
#     """
#     groups = ['^'] + list(word_groups(word)) + ['$']
#     for start in range(len(groups) - 2):
#         yield ''.join(groups[start:start + 3])


import os
import sys
import re
from languages.utils import word_triples


def count_triples(words, counters):
    rejfile = open('rejected.txt', 'w')
    for word in words:
        word = word.lower().strip()
        word = word.lstrip('*').rstrip('#')
        word = word.replace('\\', '').replace('/', '')
        word = word.replace('~', '').replace('$', '')
        word = word.replace('^', '').replace('`', '')
        word = word.replace('"', '').replace("'", '')
        if not ACCEPTABLE_WORD_REGEX.match(word):
            print >> rejfile, word
            continue
        for triple in word_triples(word):
            counters[triple] = counters.get(triple, 0) + 1
    rejfile.close()


if __name__ == '__main__':
    counters = {}
    if len(sys.argv) > 1:
        for filename in sys.argv[1:]:
            count_triples(open(filename), counters)
    else:
        count_triples(sys.stdin, counters)
    pairs = [(counters[triple], triple) for triple in counters]
    pairs.sort(reverse=True)
    pairs = pairs[:20000]
    print 'TRIPLE_SCORES = {}'

# Path: languages/utils.py
# def word_triples(word):
#     """
#     >>> list(word_triples('weight'))
#     ['^wei', 'weight', 'eight$']
#     >>> list(word_triples('eightyfive'))
#     ['^eight', 'eighty', 'ghtyf', 'yfi', 'fiv', 'ive', 've$']
#     """
#     groups = ['^'] + list(word_groups(word)) + ['$']
#     for start in range(len(groups) - 2):
#         yield ''.join(groups[start:start + 3])


# Path: tools/languages_analyze_wordlist.py
import os
import sys
import re
from languages.utils import word_triples



def count_triples(words, counters):
    rejfile = open('rejected.txt', 'w')
    for word in words:
        word = word.lower().strip()
        word = word.lstrip('*').rstrip('#')
        word = word.replace('\\', '').replace('/', '')
        word = word.replace('~', '').replace('$', '')
        word = word.replace('^', '').replace('`', '')
        word = word.replace('"', '').replace("'", '')
        if not ACCEPTABLE_WORD_REGEX.match(word):
            print >> rejfile, word
            continue
        for triple in word_triples(word):
            counters[triple] = counters.get(triple, 0) + 1
    rejfile.close()


if __name__ == '__main__':
    counters = {}
    if len(sys.argv) > 1:
        for filename in sys.argv[1:]:
            count_triples(open(filename), counters)
    else:
        count_triples(sys.stdin, counters)
    pairs = [(counters[triple], triple) for triple in counters]
    pairs.sort(reverse=True)
    pairs = pairs[:20000]
    print 'TRIPLE_SCORES = {}'

    print 'for index, triple in enumerate("""'