<commit_before>#! /usr/bin/env python
"""
Usage:
  parse_text.py books <books>...

Options:
  -h --help     Show this screen.
  --version     Show version.
"""
from docopt import docopt
from collections import defaultdict


ARGUMENTS = docopt(__doc__, version='parse text 1.0')

BOOKS = ARGUMENTS['<books>']

WORDS_USED = defaultdict(list)
for i, book in enumerate(BOOKS):
    with open(book+'.txt') as f:
        CONTENT = f.readlines()

    for line_num, line in enumerate(CONTENT):
        for word in line.split():
            word = word.strip("',.;:?!").title()
            if word.isalpha() is False:
                print word
            WORDS_USED[word].append('{}.{}'.format(i+1, line_num+1))

with open('out.dat', 'w') as f:
    f.write('word, times used, lines used\n')
    for key, value in sorted(WORDS_USED.items()):
        f.write('{}, {}, {}\n'.format(key, len(value), value))<commit_msg>use input file not command line to specify files<commit_after>#! /usr/bin/env python
"""
Usage:
  parse_text.py input <input>

Options:
  -h --help     Show this screen.
  --version     Show version.
"""
from docopt import docopt
from collections import defaultdict
import yaml


ARGUMENTS = docopt(__doc__, version='parse text 1.0')

with open('{}.yml'.format(ARGUMENTS['<input>']), 'r') as f:
    INPUT_DICT = yaml.load(f)

BOOKS = INPUT_DICT['books']
WORDS_USED = defaultdict(list)

for i, book in enumerate(BOOKS):
    with open(book+'.txt') as f:
        CONTENT = f.readlines()

    for line_num, line in enumerate(CONTENT):
        for word in line.split():
            word = word.strip("',.;:?!").title()
            if word.isalpha() is False:
                print word
            WORDS_USED[word].append('{}.{}'.format(i+1, line_num+1))

with open('out.dat', 'w') as f:
    f.write('word, times used, lines used\n')
    for key, value in sorted(WORDS_USED.items()):
        f.write('{}, {}, {}\n'.format(key, len(value), value))
{'fraction_non_alphanumeric': 0.1668377823408624, 'fraction_numerical': 0.004106776180698152, 'mean_word_length': 2.9613821138211383, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '860775', 'n_tokens_mistral': 677, 'n_tokens_neox': 656, 'n_words': 180}