<jupyter_start><jupyter_text>Data<jupyter_code>import pandas as pd
df = pd.read_csv('../data/clean_can.csv')
df.head()<jupyter_output><empty_output><jupyter_text>Tokenize<jupyter_code>from sklearn.feature_extraction.text import CountVectorizer, TfidfVectorizer
from sklearn.metrics.pairwise import cosine_similarity
from sklearn.neighbors import NearestNeighbors
from sklearn.decomposition import PCA

from spacy.tokenizer import Tokenizer
import spacy
nlp = spacy.load("en_core_web_md")
# Instantiate tokenizer
tokenizer = Tokenizer(nlp.vocab)
# Define Stop Words
STOP_WORDS = nlp.Defaults.stop_words.union([' ', '  ', 'the', 'like', 'i', 'for', 'I', "i've", 'we', '\n', '\n\n', 'my', 'this'])
# Define tokenize function
def tokenize(series):
    for doc in tokenizer.pipe(series, batch_size = 500):
        # Initialize doc tokens
        doc_tokens = []

        # Filter for stop words
        for token in doc:
            if token.text not in STOP_WORDS:
                doc_tokens.append(token.text.lower())

        # Append doc_tokens to tokens
        tokens.append(doc_tokens)
    return tokens
# Apply tokenize function
tokens = []
tokens = tokenize(df['Flavor'])
df['Tokens'] = tokens
# Loop to clean up
for token_list in tokens:
    for token in token_list:
        if token in STOP_WORDS:
            token_list.remove(token)<jupyter_output><empty_output><jupyter_text>Vector Representation<jupyter_code># Clean up text
clean_descriptions = []

# Loop over tokens
for token_list in df['Tokens']:
    clean_description = ' '.join(token_list)
    clean_descriptions.append(clean_description)
# Send to X
df['clean_descriptions'] = clean_descriptions
# Instantiate Vectorizer
tfidf = TfidfVectorizer(stop_words = 'english')
tfidf = tfidf.fit(df['clean_descriptions'])
# Pickle tfidf Vectorizer
filename = '../models/flavor_vectorizer_01.pkl'
pickle.dump(tfidf, open(filename, 'wb'))
# Create vocab and get word counts
sparse = tfidf.transform(df['clean_descriptions'])
# send the matrix to a DataFrame
tfidf_dtm = pd.DataFrame(sparse.todense(), columns = tfidf.get_feature_names())<jupyter_output><empty_output><jupyter_text>KNN Model<jupyter_code># Instantiate nearest neighbors model
flavor_nn = NearestNeighbors(n_neighbors=5, algorithm='ball_tree')
flavor_nn.fit(tfidf_dtm)<jupyter_output><empty_output><jupyter_text>Make Predictions (API)<jupyter_code>import pickle
import pandas as pd

from sklearn.feature_extraction.text import TfidfVectorizer
# Import model and Vectorizer
flavor_knn = pickle.load(open("../models/flavor_knn_01.pkl", 'rb'))
tfidf = pickle.load(open("../models/flavor_vectorizer_01.pkl", "rb"))
# Get data
df = pd.read_csv('../data/clean_can.csv')
# Create a fake weed review
fake_input = """flowery, violet, diesel"""
def get_5_recommendations(request):
    # Transform
    request = pd.Series(request)
    request_sparse = tfidf.transform(request)

    # Send to df
    request_tfidf = pd.DataFrame(request_sparse.todense())

    # Return a list of indexes
    top5 = flavor_knn.kneighbors([request_tfidf][0], n_neighbors=5)[1][0].tolist()
    
    # Send recomendations to DataFrame
    recommendations_df = df.iloc[top5]
    
    return recommendations_df
# Test request function
top5 = get_5_recommendations(fake_input)
top5<jupyter_output><empty_output><jupyter_text>Pickle Model<jupyter_code>import pickle
# Export Pickle File
filename = '../models/flavor_knn_01.pkl'
pickle.dump(flavor_nn, open(filename, 'wb'))<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1153184165232358, 'fraction_numerical': 0.006310958118187034, 'mean_word_length': 5.19360568383659, 'pattern_counts': {'":': 0, '<': 25, '<?xml version=': 0, '>': 25, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '351768', 'license': 'MIT', 'n_tokens_mistral': 1238, 'n_tokens_neox': 1198, 'n_words': 252, 'path': 'modelling/knn_flavor_model.ipynb', 'repo_name': 'Build-Week-Med-Cabinet/Data-Science', 'size': '3486'}