<jupyter_start><jupyter_text>Read in cannabis.csv<jupyter_code>import pandas as pd

URL = "https://raw.githubusercontent.com/ekselan/DS-2/master/data/cannabis.csv"
cancsv = pd.read_csv(URL)
print(cancsv.shape)
cancsv.head()<jupyter_output>(2351, 6)<jupyter_text>Create csv dataset from The Strain API<jupyter_code># The Strain API Key

API_KEY = "iZMkqry"
# Import statements for interaction with API
import numpy as np
import requests
import json
import requests
response = requests.get(f"https://strainapi.evanbusse.com/{API_KEY}/searchdata/effects")
print(response.json())<jupyter_output>[{'effect': 'Relaxed', 'type': 'positive'}, {'effect': 'Dizzy', 'type': 'negative'}, {'effect': 'Hungry', 'type': 'positive'}, {'effect': 'Euphoric', 'type': 'positive'}, {'effect': 'Happy', 'type': 'positive'}, {'effect': 'Depression', 'type': 'medical'}, {'effect': 'Insomnia', 'type': 'medical'}, {'effect': 'Pain', 'type': 'medical'}, {'effect': 'Stress', 'type': 'medical'}, {'effect': 'Cramps', 'type': 'medical'}, {'effect': 'Creative', 'type': 'positive'}, {'effect': 'Energetic', 'type': 'positive'}, {'effect': 'Talkative', 'type': 'positive'}, {'effect': 'Lack of Appetite', 'type': 'medical'}, {'effect': 'Nausea', 'type': 'medical'}, {'effect': 'Dry Mouth', 'type': 'negative'}, {'effect': 'Headache', 'type': 'medical'}, {'effect': 'Uplifted', 'type': 'positive'}, {'effect': 'Tingly', 'type': 'positive'}, {'effect': 'Paranoid', 'type': 'negative'}, {'effect': 'Sleepy', 'type': 'positive'}, {'effect': 'Fatigue', 'type': 'medical'}, {'effect': 'Headaches', 'type': 'medical'}, {'effec[...]<jupyter_text>Read in data from API<jupyter_code># API call for all strains
url = f"https://strainapi.evanbusse.com/{API_KEY}/strains/search/all"
JSONContent = requests.get(url).json()
content = json.dumps(JSONContent, indent=3, sort_keys=True) #> Just to print and
# investigate
# print(content)
type(JSONContent)
df = pd.DataFrame.from_dict(JSONContent)
# Transpose for proper shape
df = df.T
# Reset Index
df = df.reset_index()
# Rename old index column
df = df.rename(columns={"index":"strain"})
print(df.shape)
# df.head()
df2 = pd.json_normalize(df['effects'])
# df2.head()
type(df2['positive'][0])
# Make copy of dataset
data = df.copy()
print(data.shape)
# data.head()
# df with effects dictionaries unpacked
df2 = data['effects'].apply(pd.Series)
# Establish foreign key to merge on
df2['id'] = data['id']
print(df2.shape)
# df2.head()
# Merge unpacked effects into data
dataset = data.merge(df2, how='inner', on='id')
dataset = dataset.drop('effects', axis=1)
print(dataset.shape)
dataset.head()
dataset.isnull().sum()<jupyter_output><empty_output><jupyter_text>Prep data for merge<jupyter_code># Function to wrangle column values

def wrangle(x):
  """ Wrangle column value lists into strings"""
  # flavors column
  flavors = []
  for row in x['flavors']:
    row = str(row)
    row = row.replace("[","") 
    row = row.replace("'","") 
    row = row.replace("]","") 
    flavors.append(row)
  x['flavors'] = flavors

  # positive column
  pos = []
  for row in x['positive']:
    row = str(row)
    row = row.replace("[","") 
    row = row.replace("'","") 
    row = row.replace("]","")
    pos.append(row)
  x['positive'] = pos

  # negative column
  neg = []
  for row in x['negative']:
    row = str(row)
    row = row.replace("[","") 
    row = row.replace("'","") 
    row = row.replace("]","")
    neg.append(row)
  x['negative'] = neg

  # medical column
  med = []
  for row in x['medical']:
    row = str(row)
    row = row.replace("[","") 
    row = row.replace("'","") 
    row = row.replace("]","")
    med.append(row)
  x['medical'] = med
  return x
dataset_clean = dataset.copy()
dataset_clean = wrangle(dataset_clean)
dataset_clean.head()<jupyter_output><empty_output><jupyter_text>Merge cannabis.csv with The Strain data<jupyter_code>print('TheStrain:', dataset_clean.shape)
print('cancsv:', cancsv.shape)
cancsv = cancsv.rename(columns={"Strain":"strain"})
# cancsv.head()
# Clean strings in strain column to prep for merge

def wrangle_cancsv(x):
  rows = []
  for row in x['strain']:
    row = row.replace("-"," ")
    rows.append(row)
  x['strain'] = rows
  return x

can_clean = wrangle_cancsv(cancsv)
# Merge data
final = dataset_clean.merge(cancsv, how='inner', on='strain')
df = final.copy()
df = df.drop(columns=['race','Effects','Description','negative'])
# Replace empty values with nan
df = df.replace(r'^\s*$', np.nan, regex=True)
df = df.dropna()
print(df.shape)
df.head()
df.isnull().sum()<jupyter_output><empty_output><jupyter_text>Export dataset to .csv<jupyter_code># df.to_csv("/files/BW_MedCab_Dataset.csv", index=False)
# df.to_csv("/files/BW_MedCab_Dataset_With_Index.csv", index=True)<jupyter_output><empty_output><jupyter_text>NLP  Tokenize Effects<jupyter_code>import spacy
!python -m spacy download en_core_web_sm
from spacy.tokenizer import Tokenizer

nlp = spacy.load('en_core_web_sm')

STOP_WORDS = nlp.Defaults.stop_words.union(["of"])
text = str(df['medical'])

doc = nlp(text)

# Tokenizer
tokenizer = Tokenizer(nlp.vocab)

# Tokenizer Pipe
tokens = []

"""Make them tokens"""
for doc in tokenizer.pipe(df['medical'], batch_size=500):

  doc_tokens = []

  for token in doc:
    if token.text.lower() not in STOP_WORDS:
      doc_tokens.append(token.text.lower())

  tokens.append(doc_tokens)

df['tokens'] = tokens
type(tokens)
df['tokens'][0]<jupyter_output><empty_output><jupyter_text>Get Word Counts for Medical Effects<jupyter_code>!pip install squarify
from collections import Counter
import squarify

def count(docs):
    """
    Function which takes a corpus of document and returns a dataframe
    of word counts to analyze.
    """

    word_counts = Counter()
    appears_in = Counter()

    total_docs = len(docs)

    for doc in docs:
        word_counts.update(doc)
        appears_in.update(set(doc))

    temp = zip(word_counts.keys(), word_counts.values())

    wc = pd.DataFrame(temp, columns = ['word', 'count'])

    wc['rank'] = wc['count'].rank(method='first', ascending=False)
    total = wc['count'].sum()

    wc['pct_total'] = wc['count'].apply(lambda x: x / total)

    wc = wc.sort_values(by='rank')
    wc['cul_pct_total'] = wc['pct_total'].cumsum()

    t2 = zip(appears_in.keys(), appears_in.values())
    ac = pd.DataFrame(t2, columns=['word', 'appears_in'])
    wc = ac.merge(wc, on='word')

    wc['appears_in_pct'] = wc['appears_in'].apply(lambda x: x / total_docs)

    return wc.sort_values(by='rank')
import matplotlib.pyplot as plt

wc = count(df['tokens'])

wc_top20 = wc[wc['rank'] <= 30]

squarify.plot(sizes=wc_top20['pct_total'], label=wc_top20['word'], alpha=.8 )
plt.axis('off')
plt.show()
wc.head(10)<jupyter_output><empty_output><jupyter_text>TF-IDF feature matrix<jupyter_code>from sklearn.feature_extraction.text import TfidfVectorizer


# List of text documents
data = list(df['medical'])

# create the transformer
tfidf = TfidfVectorizer(max_df=.95,
                       min_df=2,
                       ngram_range=(1,3),
                       max_features=5000)

# build vocab
dtm = tfidf.fit_transform(data) #> Similar to fit_predict

# Get feature names to use as dataframe column headers
dtm = pd.DataFrame(dtm.todense(), columns=tfidf.get_feature_names())

# View Feature Matrix as DataFrame
dtm.head()<jupyter_output><empty_output><jupyter_text>Create NearestNeighbor Model<jupyter_code>!pip install --user --upgrade scikit-learn==0.23.1
from sklearn.neighbors import NearestNeighbors

# Fit on DTM
nn = NearestNeighbors(n_neighbors=5, algorithm='ball_tree')
nn.fit(dtm)
len(data)
df.shape
data[400]
## Test NN model

symptoms = ["Tired, Sleepy, Stress, Pain"]

# Query for symptoms
new = tfidf.transform(symptoms)

nn.kneighbors(new.todense())
data[1403]
df.iloc[1403]<jupyter_output><empty_output><jupyter_text>Work on function / output<jupyter_code>def hello_neighbor():

  # Get user provided string
  x = input()

  # Query for symptoms
  new = tfidf.transform(symptoms)

  # Run model
  result = nn.kneighbors(new.todense())


  return result
symptoms = ["Tired, Sleepy, Stress, Pain"]

# Query for symptoms
new = tfidf.transform(symptoms)

result = nn.kneighbors(new.todense())
len(result)
type(result)
result
result[0]
result[1]
result[1][0][0]
result[1][0]
sorted(result[1])[0][0]
num = result[1][0][0]

info = df.iloc[num]

print(info)<jupyter_output>strain                         West Coast Dawg
id                                        2076
flavors                 Sweet, Tropical, Woody
positive    Relaxed, Euphoric, Sleepy, Aroused
medical                           Pain, Stress
Type                                    indica
Rating                                     4.8
Flavor                       Sweet,Woody,Nutty
tokens                         [pain,, stress]
Name: 1433, dtype: object<jupyter_text>Incorporate details of strain in model predict function<jupyter_code>def hello_neighbor():

  # Get user provided string
  x = input() #> this will take-in input from a route

  # Query for symptoms
  new = tfidf.transform([x])

  # Run model
  result = nn.kneighbors(new.todense())

  # Get index location of recommended strain
  num = result[1][0][0]

  # Include all details of strain except Flavor, tokens, data
  info = df.iloc[num][:7] #> could swap this to database query

  # Possibly grab top 5, loop them and grab their info

  return "Your Recommended Strain:", info
hello_neighbor()
hello_neighbor()
df.iloc[100][:7]<jupyter_output><empty_output><jupyter_text>Pickle Model<jupyter_code># Pickle (trained) model here
# Dont use model.dump use model.save instead

import pickle

# Save the trained model as a pickle string
saved_model = pickle.dumps(nn)

# Load the pickled model
nn_from_pickle = pickle.loads(saved_model)
# Pickle model with filepath for export
MODEL_FILEPATH = "/pickling/medcab_model2.pkl"

with open(MODEL_FILEPATH, "wb") as model_file:
  pickle.dump(nn, model_file)
import sklearn
sklearn.__version__
# Updated version after initially loading in pickled model to local env
# and getting warning regarding loading a pickled model from different sklearn
# version than my environment<jupyter_output><empty_output><jupyter_text>Pickled model test<jupyter_code>def hello_pickle():

  # Get user provided string
  x = input() #> this will take-in input from a route

  # Query for symptoms
  new = tfidf.transform([x])

  # Run model
  result = nn_from_pickle.kneighbors(new.todense())

  # Get index location of recommended strain
  num = result[1][0][0]

  # Include all details of strain except Flavor, tokens, data
  info = df.iloc[num][:7] #> could swap this to database query

  # Possibly grab top 5, loop them and grab their info

  return "Your Recommended Strain:", info
hello_pickle()
def hello_new_pickle():

  # Get user provided string
  x = input() #> this will take-in input from a route

  # Query for symptoms
  new = tfidf.transform([x])

  # Run model
  result = nn.kneighbors(new.todense())

  # Get index location of recommended strain
  # num = result[1][0][0]

  summary = []
  for r in result[1][0]:
    info = df.iloc[r][:7]
    summary.append(info)

  # Possibly grab top 5, loop them and grab their info

  return "Your Recommended Strain:", summary
locs = []
for r in result[1][0]:
  info = df.iloc[r][:7]
  locs.append(info)

locs
hello_new_pickle()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1538055897345755, 'fraction_numerical': 0.009404113200914044, 'mean_word_length': 4.415992384578772, 'pattern_counts': {'":': 2, '<': 55, '<?xml version=': 0, '>': 61, 'https://': 3, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '14', 'dir': 'jupyter-structured-clean-dedup', 'id': '584830', 'license': 'MIT', 'n_tokens_mistral': 4138, 'n_tokens_neox': 3934, 'n_words': 969, 'path': 'notebooks/ABW_BW_MedCab_NB1_Load_Data.ipynb', 'repo_name': 'britneh/DS', 'size': '11378'}