<jupyter_start><jupyter_text>Spcy, Spellchecker, indexer<jupyter_code>from gensim.corpora import WikiCorpus, MmCorpus, Dictionary
from nltk.tokenize import sent_tokenize, word_tokenize
import os
import random
import codecs
from nltk.corpus import stopwords
from nltk.stem.wordnet import WordNetLemmatizer
import re
from gensim.test.utils import common_texts
from gensim.corpora.dictionary import Dictionary
from gensim.models import LdaModel, TfidfModel, LsiModel
from gensim.test.utils import common_texts
from gensim.corpora.dictionary import Dictionary
import pickle
import json
from nltk.stem.wordnet import WordNetLemmatizer
from gensim.similarities import MatrixSimilarity
from gensim.models.coherencemodel import CoherenceModel
from src.topic_models import *
from gensim.models import wrappers
import matplotlib.pyplot as plt
import numpy as np
lsi_wiki = LsiWikiModel(student_tokens_path='data/student_keywords.json', wiki_tokens_path='data/wiki_keywords.pkl')
lsi_wiki.get_keywords('Sharks are cool! Fish, ocean and sharks! ')
with open('data/wiki_keywords.json', 'r') as f:
    wiki_tokens = json.load(f)
with open('data/wiki_keywords.pkl', 'w') as f:
    pickle.dumps(wiki_tokens)
# lsi_wiki.predict_lsi_index(100)
lsi_wiki = LsiWikiModel(student_tokens_path='data/student_keywords.json', wiki_tokens_path='data/wiki_keywords.pkl')
"Studentsdfd essays. sdf dasf.d"
clean_keywords = ['']
if __name__ == '__main__':
    lsi_wiki = LsiWikiModel(student_tokens_path='data/student_keywords.json', wiki_tokens_path='data/wiki_keywords.pkl')
    # while the user provides input:
    lsi_wiki.
lsi_wiki = LsiWikiModel(student_tokens_path='data/student_keywords.json', wiki_tokens_path='data/wiki_keywords.pkl')
lsi_wiki.predict_lsi(['cat', 'dog'])
# lda_keywords = LdaWikiModel(student_tokens_path='data/student_keywords.json', wiki_tokens_path='data/wiki_keywords.pkl')
with open('data/wiki_keywords.json', 'rb') as f:
    wiki_k = json.load(f)
wiki_k
with open('data/wiki_keywords.pkl', 'wb') as f:
    pickle.dump(wiki_k, f)
lsi_raw = LsiWikiModel()
lsi_raw.predict_lsi_index(35)
lsi_keywords = LsiWikiModel(student_tokens_path='data/student_keywords.json', wiki_tokens_path='data/wiki_keywords.pkl')
lsi_keywords.predict_lsi_index(10)<jupyter_output>dog know baby dog call pup puppy dog call puppy one year old dog sometimes referred man best friend kept domestic pet usually loyal like around human dog breed earliest dog breeder used wolf create domestic dog beginning human purposefully bred dog perform various task sleek greyhound type bred chase fleet football prey huge mastiff example dog bred specific job dog save day dog save day show dog police dog one water get dog save helper see even help open door help walk home home see dog lot like friend sometimes police dog wear gear dog help dog help help bye see get blind dog helper help lot help open door help keep away man take money well dog help come time smart like give money dog went get drink dog come back see dog drink want well teach dog dog become pet new evidene dog probably evolved wolf single location 20000 40000 year ago study suggests looking rate change dna oldest specimen scientist able place timing domestication dog 20000 40000 year ago train dog frist need give dog[...]
{'fraction_non_alphanumeric': 0.07081545064377683, 'fraction_numerical': 0.00827713059472716, 'mean_word_length': 6.283482142857143, 'pattern_counts': {'":': 0, '<': 4, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '125155', 'license': 'MIT', 'n_tokens_mistral': 1064, 'n_tokens_neox': 987, 'n_words': 334, 'path': 'notebooks/04_model_exploration.ipynb', 'repo_name': 'mibarguen/complementary_fact_finder', 'size': '3262'}