<jupyter_start><jupyter_text>Unit 4 Build Week - Med Cabinet - Create a NLP model that returns the top three strain reccomendations based on what the user is looking for  - Return output should be in json format- Host the database in a PostgreSQL or SQLite- Deploy a Heroku App for the front end web developers to connect to  Load and clean data<jupyter_code># Load file into Google Colab

from google.colab import files
uploaded = files.upload()
# Imports

import pandas as pd
import numpy as np

import spacy
from spacy.tokenizer import Tokenizer
from spacy.lang.en import English

from sklearn.feature_extraction import text
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.neighbors import NearestNeighbors
# Read in CSV

data = pd.read_csv("cabinet_strain.csv")
print(data.shape)
data.head()
# Cleaning the data and changing nan values to unknowns and 0s.

data = data.drop('model_id', axis=1)
data['strain_rating'] = data['strain_rating'].replace(np.nan, 0)
data['effects_profile'] = data['effects_profile'].replace(np.nan, 'Effects currently unknown')
data['flavor_profile'] = data['flavor_profile'].replace(np.nan, 'Flavor profile currently unavailable')
data['strain_description'] = data['strain_description'].replace(np.nan, 'Strain description currently unavailable')
data['strain_type'] = data['strain_type'].replace(np.nan, 'Strain type currently unknown')
# Unique Strain types

data['strain_type'] = data['strain_type'].str.capitalize()

data['strain_type'].unique()
# Unique ratings

data['strain_rating'].unique()
# Check for remaining nan values

data.isnull().sum()
# Drop nan values

data = data.copy()

data = data.dropna()
data = data.reset_index(drop=True)
data.shape
data.head()
# Create a master profile feature

data['strain_profile'] = data['strain_type'] + ',' + data['effects_profile'] + ',' + data['flavor_profile']
# Save cleaned df to use in database

# data.to_csv('med_cabinet_cleaned.csv')
# Vectorizer object

nlp=English()
tokenizer = Tokenizer(nlp.vocab)
my_words = ['unavailable', 'unknown', 'profile', 'currently']
my_stop_words = text.ENGLISH_STOP_WORDS.union(my_words)

tf = TfidfVectorizer(stop_words=my_stop_words)
# Create a data-term matrix

dtm = tf.fit_transform(data['strain_profile'].values.astype('U'))
dtm = pd.DataFrame(dtm.todense(), columns=tf.get_feature_names())
print(dtm.shape)
dtm.head()
# Define and fit model

nn = NearestNeighbors(n_neighbors=5, algorithm='ball_tree')
nn.fit(dtm)
# Test model

user01_input = ['hybrid happy strawberry']
user01_dense = tf.transform(user01_input)
_, user01_output = nn.kneighbors(user01_dense.todense())

user01_output
# Print output

list_strains = []
for n in user01_output:
    for index in n:
        list_strains.append(index)

for n in list_strains:
    print(f"{data.loc[n,:]}\n")
# # Create pickles of the model and the transformer for web deployment

# # Imports
# import pickle
# from sklearn.externals import joblib 

# # Save the model as a pickle file 
# joblib.dump(nn, 'nn02_model.pkl') 

# # Save the transformer as a pickle file
# joblib.dump(tf, 'tf_01.pkl')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11676263595649392, 'fraction_numerical': 0.007357645553422905, 'mean_word_length': 5.783080260303688, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '323543', 'license': 'MIT', 'n_tokens_mistral': 1073, 'n_tokens_neox': 1020, 'n_words': 286, 'path': 'Med_Cabinet_Final (1).ipynb', 'repo_name': 'BuildWeek-Med-Cabinet/DS', 'size': '3126'}