<jupyter_start><jupyter_text>read in, examine<jupyter_code>import pandas as pd
df = pd.read_csv('https://github.com/arewelearningyet/dashtemplate/blob/master/assets/Skyserver_12_30_2019%204_49_58%20PM.csv?raw=true')
df_alpha = df.copy() # creating backup copy
df.head().T
pd.options.display.float_format = '{:.2f}'.format
df.describe().T
df.objid.nunique()
df.specobjid.nunique()
df = df.drop(columns='specobjid')
df.describe(exclude='number')
df.shape<jupyter_output><empty_output><jupyter_text>create binary classes from multiclass feature 'class'<jupyter_code>df_alpha['class'].unique()
class_distribution = df_alpha['class'].value_counts(normalize=True).reset_index()
class_distribution.iloc[1][0]
class_distribution.iloc[1][1]
class_distribution.iloc[0]<jupyter_output><empty_output><jupyter_text>TODO: put naive baseline in here - insights about multiclass classification, 'difficulty'<jupyter_code>import plotly.graph_objects as go
colors = ['gold', 'mediumturquoise', 'darkorange', 'lightgreen']

fig = go.Figure(data=[go.Pie(labels=[class_distribution.iloc[0][0],
                                     class_distribution.iloc[1][0],
                                     class_distribution.iloc[2][0]],
                             values=[class_distribution.iloc[0][1], 
                                     class_distribution.iloc[1][1],
                                     class_distribution.iloc[2][1]])])
fig.update_traces(hoverinfo='label+percent', textinfo='text+label+percent', textfont_size=20,
                  marker=dict(colors=colors, line=dict(color='#000000', width=2,)))
fig.show()
import dash_core_components as dcc
import matplotlib.pyplot as plt
from plotly.tools import mpl_to_plotly
layout = dcc.Graph(id='class_pie', figure=fig)
df['galaxy']=df['class']=='GALAXY'
df['star'] = df['class']=='STAR'
df['quasar'] = df['class']=='QSO'
df=df.drop(columns='class')
df.quasar.value_counts()<jupyter_output><empty_output><jupyter_text>train test split<jupyter_code>from sklearn.model_selection import train_test_split
trainalpha, test = train_test_split(df)
train, val = train_test_split(trainalpha)<jupyter_output><empty_output><jupyter_text>set target<jupyter_code>target = 'quasar'

xtrain=train.drop(columns=['quasar', 'galaxy', 'star'])
ytrain=train[target]
xval=val.drop(columns=['quasar', 'galaxy', 'star'])
yval=val[target]
xtest=test.drop(columns=['quasar', 'galaxy', 'star'])
ytest=test[target]<jupyter_output><empty_output><jupyter_text>import viz libraries<jupyter_code>import matplotlib.pyplot as plt
# dash stuff
from plotly.tools import mpl_to_plotly<jupyter_output><empty_output><jupyter_text>import modeling libraries<jupyter_code># preprocesing

# pipeline<jupyter_output><empty_output><jupyter_text>linear model<jupyter_code>from sklearn.linear_model import LogisticRegressionCV#, LogisticRegression
from sklearn.preprocessing import StandardScaler
from sklearn.pipeline import make_pipeline

#model = LogisticRegression().fit(xtrain, ytrain)
pipeline = make_pipeline(
    StandardScaler(),
    LogisticRegressionCV()
)
pipeline.fit(xtrain, ytrain)
print('Validation Accuracy: ', pipeline.score(xval, yval))
lr = pipeline.named_steps['logisticregressioncv']
importances = pd.Series(lr.coef_[0], xtrain.columns)
n = 20
plt.figure(figsize=(10,n/2))
plt.title(f'Top {n} features')
importances.sort_values()[-n:].plot.barh(color='grey');
# port figure to plotly
lr = pipeline.named_steps['logisticregressioncv']
importances = pd.Series(lr.coef_[0], xtrain.columns)
n = 20
logregimports = plt.figure(figsize=(10,n/2))
plt.title(f'Top {n} features')
importances.sort_values()[-n:].plot.barh(color='grey');
pltyfig = mpl_to_plotly(logregimports)
pltyfig.show()
dump(pipeline, 'isquasarrf.joblib', compress=True)<jupyter_output><empty_output><jupyter_text>ensemble model<jupyter_code># one hot encoding with random forest
from sklearn.pipeline import make_pipeline
from sklearn.ensemble import RandomForestClassifier
%%time

print('Baseline Accuracy: ', train['quasar'].value_counts(normalize=True)[0])

pipeline=make_pipeline(
    RandomForestClassifier(random_state=0, n_jobs=-1)
)
# fit on train, score on val
pipeline.fit(xtrain, ytrain)
print('Validation Accuracy: ', pipeline.score(xval, yval))

rf = pipeline.named_steps['randomforestclassifier']
importances = pd.Series(rf.feature_importances_, xtrain.columns)
n = 20
plt.figure(figsize=(10,n/2))
plt.title(f'Top {n} features')
importances.sort_values()[-n:].plot.barh(color='grey');<jupyter_output>Baseline Accuracy:  0.89328
Validation Accuracy:  0.9917866666666667
CPU times: user 32.9 s, sys: 83.9 ms, total: 33 s
Wall time: 17 s<jupyter_text>predict galaxy<jupyter_code>target = 'galaxy'

xtrain=train.drop(columns=['quasar', 'galaxy', 'star'])
ytrain=train[target]
xval=val.drop(columns=['quasar', 'galaxy', 'star'])
yval=val[target]
xtest=test.drop(columns=['quasar', 'galaxy', 'star'])
ytest=test[target]
print('Baseline Accuracy: ', train['galaxy'].value_counts(normalize=True)[0])

pipeline=make_pipeline(
    RandomForestClassifier(random_state=0, n_jobs=-1)
)
# fit on train, score on val
pipeline.fit(xtrain, ytrain)
print('Validation Accuracy: ', pipeline.score(xval, yval))

rf = pipeline.named_steps['randomforestclassifier']
importances = pd.Series(rf.feature_importances_, xtrain.columns)
n = 20
plt.figure(figsize=(10,n/2))
plt.title(f'Top {n} features')
importances.sort_values()[-n:].plot.barh(color='grey');
#pkl
from joblib import dump

dump(pipeline, 'isgalaxyrf.joblib', compress=True)
# get versions
import joblib
import pandas
import sklearn
import matplotlib
import plotly

print(f'joblib=={joblib.__version__}')
print(f'pandas=={pandas.__version__}')
print(f'matplotlib=={matplotlib.__version__}')
print(f'plotly=={plotly.__version__}')
print(f'sklearn=={sklearn.__version__}')<jupyter_output>joblib==0.14.1
pandas==0.25.3
matplotlib==3.1.3
plotly==4.4.1
sklearn==0.22.1<jupyter_text>predict star<jupyter_code>target = 'star'

xtrain=train.drop(columns=['quasar', 'galaxy', 'star'])
ytrain=train[target]
xval=val.drop(columns=['quasar', 'galaxy', 'star'])
yval=val[target]
xtest=test.drop(columns=['quasar', 'galaxy', 'star'])
ytest=test[target]
print('Baseline Accuracy: ', train['star'].value_counts(normalize=True)[0])

pipeline=make_pipeline(
    SimpleImputer(strategy='median'),
    RandomForestClassifier(random_state=0, n_jobs=-1)
)
# fit on train, score on val
pipeline.fit(xtrain, ytrain)
print('Validation Accuracy: ', pipeline.score(xval, yval))

dump(pipeline, 'isstarrf.joblib', compress=True)

rf = pipeline.named_steps['randomforestclassifier']
importances = pd.Series(rf.feature_importances_, xtrain.columns)
n = 20
plt.figure(figsize=(10,n/2))
plt.title(f'Top {n} features')
importances.sort_values()[-n:].plot.barh(color='grey');<jupyter_output>Baseline Accuracy:  0.6211377777777778
Validation Accuracy:  0.9980266666666666<jupyter_text>viz<jupyter_code>plt.scatter(df_alpha['class'], df_alpha['redshift'])<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.15264440303855525, 'fraction_numerical': 0.023649132865128278, 'mean_word_length': 7.297265160523187, 'pattern_counts': {'":': 0, '<': 46, '<?xml version=': 0, '>': 46, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 3, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '12', 'dir': 'jupyter-structured-clean-dedup', 'id': '347039', 'license': 'MIT', 'n_tokens_mistral': 2644, 'n_tokens_neox': 2525, 'n_words': 382, 'path': 'notebooks/sloan.ipynb', 'repo_name': 'CSLSDS/SkySurvey', 'size': '6977'}