<jupyter_start><jupyter_text>Med Cab Unit 4 NotebookNotebook for Data analysis for DS17 Unit 4 Build week<jupyter_code># Imports
import pandas as pd
import numpy as np
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.neighbors import NearestNeighbors
import en_core_web_sm
import spacy
from spacy import load
url = 'https://raw.githubusercontent.com/build-week-ft-med-cabinet-3/datascience/master/data/toking.csv'

df = pd.read_csv(url)

df.head()
df.shape<jupyter_output><empty_output><jupyter_text>List of TODOs:- Tokenze relevant columns  - Are we just using ailments?  Or do we also want to include Positive effects?  Negative?- Nearest Neighbors model  - Initially will only play with ailments  - Explore the `search` column<jupyter_code>df['ailment'] = df['ailment'].astype(str)
for tokens in df['ailment']:
  if type(tokens) == float:
    print(tokens)
# Tokenize columns
nlp= en_core_web_sm.load()
def tokenizer(text):
    doc=nlp(str(text).lower())
    return [token.lemma_ for token in doc if ((token.is_stop == False) and
    (token.is_punct == False)) and (token.pos_ != 'PRON')]
df['ailment_tokens'] = df['ailment'].apply(tokenizer)
df.head()
df.dropna(inplace=True)
tokenset = set()

for tokens in df['ailment_tokens']:
  for token in tokens:
    tokenset.add(token)
tokenset<jupyter_output><empty_output><jupyter_text>Combining datasetsImporting the csv that contains descriptions for strains and adding them to the existing dataset that contains ailments.<jupyter_code>url2 = 'https://raw.githubusercontent.com/med-cab-1/data_engineer/master/Data/cannabis_new.csv'

df2 = pd.read_csv(url2)
df2.head()
df.shape
df.head()
df2['name'] = df2['Strain']
df2.reset_index()

full_df = df.merge(df2, on='name', how='left')

full_df.head(1)
full_df.shape
df = full_df[['name', 'Rating', 'type', 'ailment_tokens', 'positive_effects', 'negative_effects', 'Description', 'Effects', 'flavors']]

df.head()
df['strain_id'] = df.index
df.shape<jupyter_output><empty_output><jupyter_text>Cleaning up missing values<jupyter_code>df.isnull().sum()
df = df.replace({'nan': np.NaN})
df.isnull().sum()
df = df.dropna()
df.shape
df.isnull().sum()<jupyter_output><empty_output><jupyter_text>Clean up the dataset- Ensure all column titles are lower case<jupyter_code>df.columns = [x.lower() for x in df.columns]
df.head()<jupyter_output><empty_output><jupyter_text>Output the final cleaned set into a csv for use in creating the local db<jupyter_code>df.to_csv('cannabis_final.csv')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13177681044717054, 'fraction_numerical': 0.00593589236248516, 'mean_word_length': 6.243553008595988, 'pattern_counts': {'":': 0, '<': 25, '<?xml version=': 0, '>': 25, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '437535', 'license': 'MIT', 'n_tokens_mistral': 926, 'n_tokens_neox': 903, 'n_words': 213, 'path': 'notebooks/medcab_data_analysis.ipynb', 'repo_name': 'JeremySpradlin/datascience', 'size': '2527'}