<jupyter_start><jupyter_text>Text Features:<jupyter_code>df['Savor'] = ', ' + df['Flavor'].astype(str)
df['Style'] = ', ' + df['Description'].astype(str)
df['Breed'] = ', ' + df['Strain'].astype(str)
df['Text'] = df['Effects'].map(str) + df['Savor']
df['Text'] = df['Text'].map(str) + df['Style']
df['Text'] = df['Text'].map(str) + df['Breed']
df['Text'] = df['Text'].map(str) + df['Style']
df['Effects_and_Flavor'] = df['Effects'].map(str) + df['Savor']
df.head(3)
df = df.reset_index()
df['ID'] = df['Strain'].index
df['ID'] = df['ID'].astype(int)
df['ID'].isna().sum()
np.random.seed(42)
df.shape
df['index'].nunique(), df['ID'].nunique()
df['Text']= df['Text'].str.replace(',', ' ')<jupyter_output><empty_output><jupyter_text>Vectorizing:<jupyter_code>df.head(2)
df.drop(['Unnamed: 0'], axis=1).head()
# possible analysis functions

def count(docs):

        word_counts = Counter()
        appears_in = Counter()
        
        total_docs = len(docs)

        for doc in docs:
            word_counts.update(doc)
            appears_in.update(set(doc))

        temp = zip(word_counts.keys(), word_counts.values())
        
        wc = pd.DataFrame(temp, columns = ['word', 'count'])

        wc['rank'] = wc['count'].rank(method='first', ascending=False)
        total = wc['count'].sum()

        wc['pct_total'] = wc['count'].apply(lambda x: x / total)
        
        wc = wc.sort_values(by='rank')
        wc['cul_pct_total'] = wc['pct_total'].cumsum()

        t2 = zip(appears_in.keys(), appears_in.values())
        ac = pd.DataFrame(t2, columns=['word', 'appears_in'])
        wc = ac.merge(wc, on='word')

        wc['appears_in_pct'] = wc['appears_in'].apply(lambda x: x / total_docs)
        
        return wc.sort_values(by='rank')
#Tokenizer
tokenizer = Tokenizer(nlp.vocab)
#Tokenizer Pipe

tokens = []

""" Make them tokens """
for doc in tokenizer.pipe(df['Text'], batch_size=100):
    doc_tokens = [token.text for token in doc]
    tokens.append(doc_tokens)

df['tokens'] = tokens
# Using the "Count" function
wc = count(df['tokens'])
wc = count(df['tokens'])
wc.head(3)
wc_top20 = wc[wc['rank'] <= 30]

squarify.plot(sizes=wc_top20['pct_total'], label=wc_top20['word'], alpha=.8)
plt.axis('off')
plt.show()
# Object from Base Python
from collections import Counter

# The object `Counter` takes an iterable, but you can instaniate an empty one and update it. 
word_counts = Counter()

# Update it based on a split of each of our documents
df['tokens'].apply(lambda x: word_counts.update(x))

# Print out the 10 most common words
word_counts.most_common(10)
tokens = []

""" Update those tokens w/o stopwords"""
for doc in tokenizer.pipe(df['Text'], batch_size=100):
    
    doc_tokens = []
    
    for token in doc:
        if (token.is_stop == False) & (token.is_punct == False):
            doc_tokens.append(token.text.lower())

    tokens.append(doc_tokens)

df['tokens'] = tokens
wc = count(df['tokens'])
wc = count(df['tokens'])
wc.head()
wc_top20 = wc[wc['rank'] <= 30]

squarify.plot(sizes=wc_top20['pct_total'], label=wc_top20['word'], alpha=.8)
plt.axis('off')
plt.show()
wc.head(25)
wc.tail(25)
# Spacy's Default stop words:
nlp.Defaults.stop_words
wc = count(df['tokens'])
wc.head(50)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.19649984648449492, 'fraction_numerical': 0.013202333435677003, 'mean_word_length': 4.098591549295775, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '462294', 'license': 'MIT', 'n_tokens_mistral': 1243, 'n_tokens_neox': 1189, 'n_words': 232, 'path': 'Med_Cab_1.3.ipynb', 'repo_name': 'gptix/DS--Data-Engineering-', 'size': '3257'}