<jupyter_start><jupyter_text>Exploratory Data Analysis and Cleaning<jupyter_code>from sklearn.neighbors import NearestNeighbors
import pandas as pd
import spacy
from spacy.tokenizer import Tokenizer
from sklearn.feature_extraction.text import TfidfVectorizer
leafly = pd.read_csv('https://raw.githubusercontent.com/Build-Week-Med-Cabinet-6/DS/mark-dev/data/cannabis.csv')
leafly.head()
leafly.shape
leafly['Strain'].nunique()
leafly.isnull().sum()<jupyter_output><empty_output><jupyter_text>Modeling<jupyter_code># Set up spacy tokenizer


#!python -m spacy download en_core_web_lg
nlp = spacy.load('en_core_web_lg')
tokenizer = Tokenizer(nlp.vocab)

sample_text = "Text data needs to be processed to prepare it for machine learning models!"

[token.text for token in tokenizer(sample_text)]
leafly['Description'].isnull().sum()
import numpy as np
leafly.replace('None',np.NaN,inplace=True)
leafly = leafly.dropna()
# 2351 --> 2277.  I'm happy to just wholesale drop anything with missing values

leafly.shape
# Make tokens out of descriptions

tokens = []
for desc in tokenizer.pipe(leafly['Description'], batch_size=500):
    desc_tokens = [token.text for token in desc]
    tokens.append(desc_tokens)
leafly['tokens'] = tokens
leafly['tokens'].head()
leafly.head()<jupyter_output><empty_output><jupyter_text>TF-IDF Vectorizer<jupyter_code>def tokenize(document):
    
    doc = nlp(document)
    
    return [token.lemma_.strip() for token in doc if (token.is_stop != True) and (token.is_punct != True)]


# Instantiate vectorizer object
tfidf = TfidfVectorizer(stop_words='english', 
                        ngram_range=(1,2),
                        max_df=.97,
                        min_df=3,
                        tokenizer=tokenize)

# Create a vocabulary and get word counts per listing
dtm = tfidf.fit_transform(leafly['Description'])

# Get feature names to use a dataframe column headers
dtm = pd.DataFrame(dtm.todense(), columns=tfidf.get_feature_names())

# View Feature Matrix as DataFrame
dtm.head()
dtm.shape<jupyter_output><empty_output><jupyter_text>Fit a KNearestNeighbors Model on dtm<jupyter_code>#from sklearn.neighbors import NearestNeighbors

# Fit on dtm
nn = NearestNeighbors(n_neighbors=20, algorithm='kd_tree')
nn.fit(dtm)
# Query Using kneighbors 
nn.kneighbors([dtm.iloc[0]])
# user input("Describe your ideal strain!")

ideal_strain = ["""
earthy full lavored hybrid with fuit over tones
"""]
# Query the user's ideal strain. 2nd array is their top 20 strains

new = tfidf.transform(ideal_strain)
nn.kneighbors(new.todense())
!pwd
# serializ the fitted model and the fitted estimator for use in the web app
import pickle
with open('../pickles/isaac_tf.pickle','wb') as fp:
    pickle.dump(tfidf,fp)
with open('../pickles/isaac_nn.pickle','wb') as fp2:
    pickle.dump(nn,fp2)
# And then do (pseudocode)
#
# for number in nn.kneighbors(new.todense())[1]:
#     return that strain
#      Send to web so they can visualize the Top 20 list.
#


# Inspect most relevant strain
leafly['Description'][2038]
# Inspect 2nd most relevant strain
leafly['Description'][1698]
#
##
###
###     IGNORE BELOW
###
##
#
# Another dataset. Will probably ignore.

kushy = pd.read_csv('https://raw.githubusercontent.com/kushyapp/cannabis-dataset/master/Dataset/Strains/strains-kushy_api.2017-11-14.csv')
kushy.head()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11841713775662005, 'fraction_numerical': 0.01338887235941684, 'mean_word_length': 4.737201365187714, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 18, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '571299', 'license': 'MIT', 'n_tokens_mistral': 1216, 'n_tokens_neox': 1153, 'n_words': 276, 'path': 'app/ml/Build_week_IsaacGrove.ipynb', 'repo_name': 'Build-Week-Med-Cabinet-6/DS', 'size': '3361'}