<jupyter_start><jupyter_text>Metadata```Course:   DS 5001Module:   00 Final ProjectsTopic:    Using SciKit Learn to Vectorize your Corpus```  Set Up<jupyter_code>data_home = "../data"
local_lib = "../lib"
data_prefix = 'novels'
OHCO = ['book_id','chap_id','para_num','sent_num','token_num']
ngram_range = (1,4)
n_terms = 4000
import pandas as pd
import numpy as np
from sklearn.feature_extraction.text import CountVectorizer, TfidfVectorizer, TfidfTransformer<jupyter_output><empty_output><jupyter_text>Import CORPUS<jupyter_code>LIB = pd.read_csv(f"{data_home}/{data_prefix}/{data_prefix}-LIB.csv").set_index(OHCO[:1])
CORPUS = pd.read_csv(f"{data_home}/{data_prefix}/{data_prefix}-CORPUS.csv").set_index(OHCO)
CORPUS.head()<jupyter_output><empty_output><jupyter_text>Create DOC<jupyter_code>def gather_docs(CORPUS, ohco_level, term_col='term_str'):
    OHCO = CORPUS.index.names
    CORPUS[term_col] = CORPUS[term_col].astype('str')
    DOC = CORPUS.groupby(OHCO[:ohco_level])[term_col].apply(lambda x:' '.join(x)).to_frame('doc_str')
    return DOC
DOC = gather_docs(CORPUS, 2)
DOC['n_tokens'] = DOC.doc_str.apply(lambda x: len(x.split()))
DOC<jupyter_output><empty_output><jupyter_text>Method 1: CountVectorizer + TfidfTransformer  Create DTM<jupyter_code>count_engine = CountVectorizer(
    stop_words='english',
    ngram_range=ngram_range,
    max_features=n_terms)
X = count_engine.fit_transform(DOC.doc_str)
# X.toarray()
# count_engine.get_feature_names()
DTM = pd.DataFrame(X.toarray(), 
                   columns=count_engine.get_feature_names(), 
                   index=DOC.index)
DTM<jupyter_output><empty_output><jupyter_text>Get VOCAB<jupyter_code>VOCAB = DTM.sum().to_frame('n')
VOCAB .sort_index()
VOCAB['n_chars'] = VOCAB.apply(lambda x: len(x.name), 1)
VOCAB['n_tokens'] = VOCAB.apply(lambda x: len(x.name.split()), 1)
VOCAB.value_counts('n_tokens')
VOCAB[VOCAB.n_tokens == 2]<jupyter_output><empty_output><jupyter_text>Create TFIDF<jupyter_code># TfidfTransformer?
tfidf_engine = TfidfTransformer(norm='l2', use_idf=True)
X1 = tfidf_engine.fit_transform(DTM)
TFIDF = pd.DataFrame(X1.toarray(), columns=DTM.columns, index=DTM.index)
TFIDF<jupyter_output><empty_output><jupyter_text>Add stats to VOCAB<jupyter_code>VOCAB['tfidf_mean'] = TFIDF.mean()
VOCAB['df'] = DTM[DTM > 0].count()
VOCAB['dfidf'] = VOCAB.df * np.log2(len(TFIDF)/VOCAB.df)
VOCAB.sort_values('dfidf', ascending=False).head(10)<jupyter_output><empty_output><jupyter_text>Method 2: TfidfVectorizerThis method combines the two.<jupyter_code># TfidfVectorizer?<jupyter_output><empty_output><jupyter_text>Create TFIDF<jupyter_code>tfidf_engine2 = TfidfVectorizer(
    stop_words='english',
    ngram_range=ngram_range,
    max_features=n_terms,
    norm='l2', 
    use_idf=True)
X2 = tfidf_engine2.fit_transform(DOC.doc_str)
TFIDF2 = pd.DataFrame(X2.toarray(), columns=tfidf_engine2.get_feature_names(), index=DTM.index)
TFIDF2<jupyter_output><empty_output><jupyter_text>Get VOCAB<jupyter_code>VOCAB2 = TFIDF2.mean().to_frame('tfidf_mean')
VOCAB2<jupyter_output><empty_output><jupyter_text>Add stats to VOCAB<jupyter_code>VOCAB2['df'] = TFIDF2[TFIDF2 > 0].count()
VOCAB2['dfidf'] = VOCAB2.df * np.log2(len(TFIDF2)/VOCAB2.df)
VOCAB2.sort_values('dfidf', ascending=False).head(10)<jupyter_output><empty_output><jupyter_text>Create BOW<jupyter_code>BOW = DTM[DTM > 0].stack().to_frame('n').join(TFIDF[TFIDF > 0].stack().to_frame('tfidf'))
BOW<jupyter_output><empty_output><jupyter_text>Play<jupyter_code>DOC['n_tokens_used'] = BOW.groupby(OHCO[:2]).n.sum().astype('int')
DOC['token_use_rate'] = DOC.n_tokens_used / DOC.n_tokens
DOC
DOC.sort_values('token_use_rate').token_use_rate.plot(rot=45, figsize=(10,5));
DOC.loc['adventures'].token_use_rate.plot(style='o-');
DOC.loc['udolpho'].token_use_rate.plot(style='o-');
DOC.loc['northangerabbey'].token_use_rate.plot(style='o-');
LIB['token_use_rate_mean'] = DOC.groupby('book_id').token_use_rate.mean()
LIB['n_tokens'] = DOC.groupby('book_id').n_tokens.sum()
LIB.sort_values('token_use_rate_mean').token_use_rate_mean.plot.barh(figsize=(5,7));
LIB.sort_values('token_use_rate_mean').style.background_gradient()
LIB.plot.scatter('n_tokens', 'token_use_rate_mean', logx=False);<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.16147308781869688, 'fraction_numerical': 0.01392823418319169, 'mean_word_length': 8.43652561247216, 'pattern_counts': {'":': 0, '<': 53, '<?xml version=': 0, '>': 57, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '13', 'dir': 'jupyter-structured-clean-dedup', 'id': '118519', 'license': 'MIT', 'n_tokens_mistral': 1841, 'n_tokens_neox': 1762, 'n_words': 184, 'path': 'M00_FinalProjects/M00_01_VectorizationWithSKLearn.ipynb', 'repo_name': 'avnimalik/DS5001-2022-01', 'size': '4236'}