<jupyter_start><jupyter_text><jupyter_code># read in dataset 1: pokemon Go character sightings

import pandas as pd
#import zipfile

#predictemall.read_csv
#predictemall.zip.extractall(path=None, members=None, pwd=None)
df = pd.read_csv('300k.csv')
pd.set_option("display.max_rows", 91835)
#pd.set_option("display.max_columns", 300)
df

df.shape
df.head()
# read in dataset 2: encoded names and corresponding id#s
df_Names = pd.read_csv('pokemonNumbers.csv', header= None)
pd.set_option("display.max_rows", 91835)
df_Names.columns=['pokemonId', 'pokemonName'] # change df_Names column names to align with the proper columns for merge.

df_Names
df_Names.head(20)
df_Names.shape
#EXPLORATORY DATA ANALYSIS/DATA WRANGLING
'''
join(df2) does a left join by default (keeps all rows of df1 ), but df. merge does an inner join by default (returns only matching rows of df1 and df2 ).Jun 18, 2016

What is the difference between join and merge in Pandas? - Stack ...stackoverflow.com › questions › what-is-the-di

# merge df and df_Names to make df3
df3 =  pd.merge(df_Names, df, on = 'pokemonId')
pd.set_option("display.max_rows", 91835)
df3
'''
# try df.join(other.set_index('key'), on='key')
df3 = df.join(df_Names.set_index('pokemonId'), on='pokemonId')
df3
df3.head()<jupyter_output><empty_output><jupyter_text>**check if merge preserved pokemonId:pokemonName**<jupyter_code># ATTEMPT 1: locate row where latitude = 20.525745, longitude =	-97.460829 to check if merge preserved pokemonId:pokemonName
#df.iloc[lambda x: df3['latitude'] == 20.525745] - GET: NotImplementedError: iLocation based boolean indexing on an integer type is not available
# ATTEMPT 2: locate row of before and after merge..
#df.query('latitude = 20.525745 and longitude = -97.460829')
# ATTEMPT 3: df.loc[df['column_name'] == some_value]
#df.loc[df['latitude'] == 20.525745] and df.loc[df['longitude'] == -97.460829]

'''
To select rows whose column value equals a scalar, some_value, use ==:
Combine multiple conditions with &:

df.loc[(df['column_name'] >= A) & (df['column_name'] <= B)]
'''
#USE LOCATED ROW TO COMPARE w/col vals in df3 
df.loc[(df['latitude'] == 20.525745) & (df['longitude'] == -97.460829) & (df['pokemonId'] == 19)] 
# try to locate and compare with analogous row in df3- found NO ROWS
#df3.loc[(df3['latitude'] == 20.525745) & (df3['longitude'] == -97.460829) & (df3['pokemonName'] == 'Raticate')] - FOUND NO ROWS

#RETURNS A ROW BUT ROW INDEX OFF BY 1; "Raticate" is id# 20, but queried pokemonId was 19 in df_Names - MAY FIX LATER
df3.loc[(df3['latitude'] == 20.525745) & (df3['longitude'] == -97.460829) & (df3['pokemonId'] == 19)]
#  ATTEMPT 4: 
df.loc[df['latitude'] == 20.525745]
# MERGED DATAFRAME df3... ROW INDEX OFF BY 1; "Raticate" is id# 20 in df_Names - fix by dropping pokemonId as 300k.csv dataset creator advised

#df.drop(['B', 'C'], axis=1)- SAMPLE
#df.drop(['pokemonId'], axis=1) - DOESN'T WORK ?!#$

#df.drop(columns=['B', 'C'])- SAMPLE
df.drop(columns=['pokemonId'])
df_Names.drop(columns=['pokemonId'])  #- NOT DROPPED EITHER; MAYBE IT'S A TUPLE?..

#df_Names.dtypes()
df_Names.head(5)

# STILL CAN'T DROP 'pokemonID' from ANY df!! 
'''
df3.drop(columns=['pokemonId']) 
df3.head(5)
'''
#Choosing'appearedDayOfWeek' as target
df['appearedDayOfWeek'].nunique()
df['appearedDayOfWeek'].value_counts()
#df3['appearedDayOfWeek'].nunique()<jupyter_output><empty_output><jupyter_text>if df['appearedDayOfWeek'] changes from 7 next time I reload file(s), I will CHOOSe the NEW TARGET TO PREDICT to be 'Continent' (nunique=11?!)!<jupyter_code># check df Nulls- returns 0 for all columns
df.isnull().sum()
# check df_Names Nulls- returns 0 for all columns
df_Names.isnull().sum()
# check df3 Nulls- returns 0 for all columns
df3.isnull().sum()
# THIS TIME AROUND GETTING NO NULLS...
#drop df cols with nulls
#df.dropna() - not work right
'''
df.dropna(axis='columns') # also not work right
df
'''
#df.columns.to_list()
#View correlation matrix
df.corr()
# THIS TIME AROUND GETTING NO NULLS...
#trim df to reduce noise including nulls
'''
df_trimmed = df.drop(columns=['gymIn100m',
 'gymIn250m',
 'gymIn500m',
 'gymIn1000m',
 'gymIn2500m',
 'gymIn5000m',
 'cooc_114',
 'cooc_115',
 'cooc_116',
 'cooc_117',
 'cooc_118',
 'cooc_119',
 'cooc_120',
 'cooc_121',
 'cooc_122',
 'cooc_123',
 'cooc_124',
 'cooc_125',
 'cooc_126',
 'cooc_127',
 'cooc_128',
 'cooc_129',
 'cooc_130',
 'cooc_131',
 'cooc_132',
 'cooc_133',
 'cooc_134',
 'cooc_135',
 'cooc_136',
 'cooc_137',
 'cooc_138',
 'cooc_139',
 'cooc_140',
 'cooc_141',
 'cooc_142',
 'cooc_143',
 'cooc_144',
 'cooc_145',
 'cooc_146',
 'cooc_147',
 'cooc_148',
 'cooc_149',
 'cooc_150',
 'cooc_151',
 ])
 '''
#trim df to reduce noise:
''' 1st do
 1) CORRELATION MATRIX & 
 2) HEATMAP to try to determine which columns we may drop that may not or
   not significantly affect predicting TARGET = 'continent' 
'''

df3['continent'].nunique()
df3['continent'].value_counts()
df['continent'].nunique()
#df_trimmed.sort_by('pokemonId', ascending = True)
#check data distribution to see if linear model is appropriate- IT IS NOT; THIS IS MULTI-CLASS CLASSIFICATiON PROBLEM
df_trimmed.plot.hist() 
#Get BASELINE- the first way to get the highest accuracy by assuming majority, [by] assum[ing] majority:

pd.set_option('display.max_columns', 500)
pd.set_option('display.max_row', 300000)
'''
average for regression should be the majority value in a sense; 
classification is assuming all values are the highest majority value- BASELINE IS PREDICTING MODE CLASS (largest # of observations
'''

df['appearedDayOfWeek'].value_counts()
# our BASELINE IS SATURDAY, WITH 72201 appearances on that day
df['appearedDayOfWeek'].nunique()
# Fit a Classification model
# Try a shallow decision tree as a fast, first model

#pip install --upgrade category_encoders
# from sklearn.preprocessing import CategoricalEncoder
#pip install category_encoders

!pip install category_encoders==2.* 

import numpy as np
import category_encoders as ce
from sklearn.pipeline import make_pipeline
from sklearn.impute import SimpleImputer
from sklearn.tree import DecisionTreeClassifier
from sklearn.ensemble import RandomForestClassifier
from sklearn.metrics import roc_auc_score
from sklearn.model_selection import train_test_split

#encoder = ce.OrdinalEncoder(cols=[...])

target = 'appearedDayOfWeek'
features = ['appearedMonth', 'appearedYear', 'appearedDay', 'pokestopDistanceKm', 'weather', 'latitude', 'longitude']

#train = df 

X_train, X_test, y_train, y_test = train_test_split(df[features], df[target])
train, test = train_test_split(df)

pipeline = make_pipeline(
    ce.OrdinalEncoder(),
    SimpleImputer(strategy='most_frequent'),
    RandomForestClassifier()
)

# function to Convert 'appearedDayOfWeek' string values into floats

'''
def convertDayToFloat():

    week = list(df['appearedDayOfWeek'].replace('1.0', '2.0', '3.0','4.0','5.0','6.0','7.0'))

    print(week)

convertDayToFloat
'''    
#dictionary to Convert 'appearedDayOfWeek' string values into floats

dict = {'Sunday':'1.0', 'dummy_day':'2.0', 'Tuesday':'3.0','Wednesday':'4.0','Thursday':'5.0', 'Friday':'6.0', 'Saturday':'7.0' }
#fit model 1: Random Forest Classifier and score accuracy

pipeline.fit(X_train, y_train)
acc_score = pipeline.score(test[features], test[target])
#ra_score = roc_auc_score(test[target], pipeline.predict(test[features]))

print(f'Test Accuracy: {pipeline.score(X_test, y_test)}')
print(f'Test ROC AUC: {roc_auc_score(y_test, pipeline.predict(X_test))}\n')

'''
print(f'Val Accuracy: {acc_score}')
print(f'Val ROC AUC: {ra_score}')
'''<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.14811891607471717, 'fraction_numerical': 0.050381478558274136, 'mean_word_length': 5.686895338610378, 'pattern_counts': {'":': 0, '<': 14, '<?xml version=': 0, '>': 14, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '490919', 'license': 'MIT', 'n_tokens_mistral': 3081, 'n_tokens_neox': 2788, 'n_words': 744, 'path': '1am_SUN_Mar_8_build_2.ipynb', 'repo_name': 'LambdaTheda/DS-Unit-2-Linear-Models', 'size': '7602'}