<commit_before>#!/usr/bin/python
import sys, re, gdbm, collections
ORDER = 4
_words_regex = re.compile(r"(^\*)?[a-zA-Z',.!?\-:; ]")

def main():
    chains = collections.defaultdict(list)
    def append_chain(chain, v):
        chains[chain].append(v)
    start_offset = actions = 0
    lines_parsed = 0
    for line in sys.stdin:
        if not line: continue
        lines_parsed += 1
        if lines_parsed % 500 == 0:
            print lines_parsed
        queue, length = '', 0
        for w in _words_regex.finditer(line):
            w = w.group().lower()
            if not queue and w == ' ': continue
            if len(queue) == ORDER and not queue.startswith('*'):
                append_chain(queue, w)
            if w[-1] in '.!?':
                queue, length = '', 0
            else:
                queue += w
                length += 1
                if length == ORDER:
                    if queue.startswith('*'):
                        append_chain('__act__', queue.lstrip('*'))
                        actions += 1
                    else:
                    	if len(queue) != ORDER: print `queue`
                        append_chain('__start__', queue)
                        start_offset += 1
                queue = queue[-ORDER:]
    chains['__offset__'] = '%d;%d' % (start_offset, actions)
    chainsdb = gdbm.open(sys.argv[1], 'nf')
    for k in chains:
        chainsdb[k] = ''.join(chains[k])
    chainsdb.sync()
if __name__ == '__main__': main()<commit_msg>Fixing up makechains to actually work.<commit_after>#!/usr/bin/python
from infobat import chains
import sys

def main(dbfile):
    db = chains.Database(dbfile)
    for line in sys.stdin:
        db.learn(line.rstrip('\r\n'))
    db.sync()

if __name__ == '__main__': 
    main(sys.argv[1])

{'fraction_non_alphanumeric': 0.13854748603351955, 'fraction_numerical': 0.008938547486033519, 'mean_word_length': 1.734351145038168, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5060348', 'n_tokens_mistral': 577, 'n_tokens_neox': 543, 'n_words': 137}