<commit_before>import sys, re, cPickle
from gensim import corpora, models, similarities, utils
from gensim.corpora.dictionary import Dictionary
from gensim.corpora.textcorpus import TextCorpus
from gensim.corpora.mmcorpus import MmCorpus

import logging
logging.basicConfig(format='%(asctime)s : %(levelname)s : %(message)s', level=logging.INFO)

FILTER_WORDS = 'phonology_dict/filterWords.txt' # path to a list of words to remove

if __name__ == '__main__':
    fname = sys.argv[1]
    bfname = fname.split('.')[0]

    LEMMATIZE = utils.HAS_PATTERN
    if LEMMATIZE:
        print "you have pattern: we will lemmatize ('you were'->'be/VB')"
        suffix = '_lemmatized'
    else:
        print "you don't have pattern: we will tokenize ('you were'->'you','were')"
        suffix = '_tokenized'
    from src.prepare_corpus import tokenize
    outputname = bfname + suffix

    with open('provi' + suffix + '.ldamodel') as f:
        lda = cPickle.load(f)

    id2token = Dictionary.load_from_text('provi' + suffix + '_wordids.txt')

    out_topics = open(bfname + '_doc_topics' + suffix + '.txt', 'w')
    out_sentences = open(bfname + '_docs' + suffix + '.sin', 'w')

    if FILTER_WORDS:
        filter_words = []
        with open(FILTER_WORDS) as f:
            for line in f:
                filter_words.append(line.rstrip('\n'))<commit_msg>debug output<commit_after>import sys, re, cPickle
from gensim import corpora, models, similarities, utils
from gensim.corpora.dictionary import Dictionary
from gensim.corpora.textcorpus import TextCorpus
from gensim.corpora.mmcorpus import MmCorpus

import logging
logging.basicConfig(format='%(asctime)s : %(levelname)s : %(message)s', level=logging.INFO)

FILTER_WORDS = 'phonology_dict/filterWords.txt' # path to a list of words to remove

if __name__ == '__main__':
    fname = sys.argv[1]
    bfname = fname.split('.')[0]

    LEMMATIZE = utils.HAS_PATTERN
    if LEMMATIZE:
        print "you have pattern: we will lemmatize ('you were'->'be/VB')"
        suffix = '_lemmatized'
    else:
        print "you don't have pattern: we will tokenize ('you were'->'you','were')"
        suffix = '_tokenized'
    from src.prepare_corpus import tokenize
    outputname = bfname + suffix

    with open('provi' + suffix + '.ldamodel') as f:
        lda = cPickle.load(f)

    id2token = Dictionary.load_from_text('provi' + suffix + '_wordids.txt')

    out_topics = open(bfname + '_doc_topics' + suffix + '.txt', 'w')
    out_sentences = open(bfname + '_docs' + suffix + '.sin', 'w')

    if FILTER_WORDS:
        filter_words = []
        with open(FILTER_WORDS) as f:
            for line in f:
                filter_words.append(line.rstrip('\n'))
        filter_words = set(filter_words)
        print "the following words will be filtered", filter_words
{'fraction_non_alphanumeric': 0.1218382614891343, 'fraction_numerical': 0.0021375133594584966, 'mean_word_length': 3.313364055299539, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1056154', 'n_tokens_mistral': 976, 'n_tokens_neox': 935, 'n_words': 244}