<jupyter_start><jupyter_text>Gensim: Dictionary Generation Source: https://radimrehurek.com/gensim/wiki.html<jupyter_code>import logging
logging.basicConfig(format='%(asctime)s : %(levelname)s : %(message)s', level=logging.INFO)

from gensim import corpora, models, similarities
import re

#Import text files
import os
corpus_path = os.path.join('data')
filenames = sorted([os.path.join(corpus_path,fn) for fn in os.listdir(corpus_path) if not fn.startswith('.') ])
len(filenames)
#Data Cleaning
def remove_punc2(text):
    punctuation = '!@#$%^&*()_-+={}[]:;"\'|<>,.?/~`1234567890'
    clean_text = ""
    for character in text:
        if character not in punctuation:
            clean_text += character
    return clean_text

#Need to make a function to which can supply each document to gensim. __iter__ is the key
## Weaker impleemntation for time being: All data stored in one big matrix
## Use re
## Do it on the fly for next step


new_document = []
for n,files in enumerate(filenames):
    if files != 'data/.DS_Store':  # Find a better way to handle it.
        infile = open(files, 'r')
        text=infile.read()
        clean_text=remove_punc2(text)
        new_document.append(clean_text)

new_dictionary = corpora.Dictionary(doc.lower().split() for doc in new_document)
new_dictionary.save('/tmp/dictionary.dict')
len(new_document)
#new_document[4]
#len(new_dictionary)
stoplist=set('''court supreme in as xx xxx
                on or by that is was are were no not vs be being been has have had case 
                he she I one every least less many now ever never say says said also get
                go goes just made make put see seen whether like well back even still way
                take since another however two three four five six seven eight nine ten first second new old high 
                long and but if or because as until while  of at by for with about against
                between into through during before after above below to from up down in out
                on off over under  again further then once  here there when where why how
                all any both each few more most other some such  no nor not only own same
                so than too very what which who whom this that these those they them their
                theirs themselves it its itself she her herself he his him himself you your
                yours yourself yourselves our ours ourselves us thou we me my myself i am 
                is are was were be been being have has had having do does did doing will 
                would shall should can could may might must ought a an the'''.split())
stop_ids = [new_dictionary.token2id[stopword] for stopword in stoplist
if stopword in new_dictionary.token2id]
#non_alpha_id = [new_dictionary.token2id[nonalphaword] for nonalphaword in stoplist
#if nonalphaword.isalpha()==False]
once_ids = [tokenid for tokenid, docfreq in new_dictionary.dfs.iteritems() if docfreq == 1]

new_dictionary.filter_tokens(stop_ids + once_ids)# + non_alpha_id)
new_dictionary.compactify()
class MyCorpus(object): #Convert dictionary to vectors for low memeory
    def __iter__(self):
        for doc in new_document:
            yield new_dictionary.doc2bow(doc.lower().split())
#print [stopword for stopword in stoplist]
corpus_small_memory = MyCorpus()
corpora.MmCorpus.serialize('/tmp/sercorpusnew.mm', corpus_small_memory) #Multiple formats available
print(new_dictionary.token2id) #wihtout stopword and non-numeric characters
#print(list(corpus_small_memory))
#for vector in corpus_small_memory: # load one vector into memory at a time
#...     print len(vector)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.07858707557502738, 'fraction_numerical': 0.0052026286966046, 'mean_word_length': 3.471236230110159, 'pattern_counts': {'":': 0, '<': 6, '<?xml version=': 0, '>': 6, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '568647', 'license': 'Apache-2.0', 'n_tokens_mistral': 1091, 'n_tokens_neox': 1039, 'n_words': 412, 'path': 'Gensim_Dictionary.ipynb', 'repo_name': 'pkrouth/topic-model-poc', 'size': '3652'}