<jupyter_start><jupyter_text>Introduction to Scikit-learn<jupyter_code>import matplotlib.pyplot as plt
import numpy as np
import pandas as pd
import sklearn
from sklearn.model_selection import train_test_split

data_raw = pd.read_csv("ROC_data.csv", index_col=0)
data_raw.head()
data = data_raw.copy()
LS_type_mapping = {8: "No landslides", 1: "Fall/Topple", 2: "Torrent",   3: "Avalanche", 4: "Slump/Creep", 5: "Sackung"}
DA_area_mapping = {1: "runout", 2: "source", 3: "other"}
lithology_mapping = {1: 'Unconsolidated Sediment', 2: 'Ultrabasic rock', 3: 'Weak Metamorphic Foliated',
                     4: 'Sedimentary Rock', 5: 'Hard Metamorphic', 6: 'Intrusive Igneous',
                     7: 'Volcanic/Extrusive Igneous'}
LULC_mapping = {71: "Herbaceous", 52: "Shrubland", 41: "Forest", 31: "Barren",  21: "Developed"}

data['LS_type'] = data_raw.LS_type.map(LS_type_mapping)
data['DA_area'] = data_raw.DA_area.map(DA_area_mapping)
data['litho'] = data_raw.litho.map(lithology_mapping)
data['lulc'] = data_raw.lulc.map(LULC_mapping)
data.head()
data.shape
data.LS_type.value_counts()
# make sure none of the groups have more than 80,000 samples
# This is a bit of a silly way to subsample the "no landslide" class to make the dataset more managable
data_subsample = data.groupby('LS_type').apply(lambda x: pd.DataFrame.sample(x, min(80000, len(x)))).reset_index(drop=True)
data_subsample.LS_type.value_counts()
continuous = data_subsample.dtypes == float
continuous
import dabl
dabl.plot(data_subsample, target_col='LS_type')
X = data_subsample.drop('LS_type', axis=1)
y = data_subsample['LS_type']

X_dummy = pd.get_dummies(X)
X_dummy.head()
X_train, X_test, y_train, y_test = train_test_split(
    X_dummy, y, random_state=0)
X_train.shape
y_train.value_counts()
pd.Series(y_train).value_counts(normalize=True)<jupyter_output><empty_output><jupyter_text>Really Simple API-------------------0) Import your model class<jupyter_code>from sklearn.linear_model import LogisticRegression<jupyter_output><empty_output><jupyter_text>1) Instantiate an object and set the parameters<jupyter_code>logreg = LogisticRegression(max_iter=1000)<jupyter_output><empty_output><jupyter_text>2) Fit the model<jupyter_code>logreg.fit(X_train, y_train)<jupyter_output>/home/andy/checkout/scikit-learn/sklearn/linear_model/_logistic.py:762: ConvergenceWarning: lbfgs failed to converge (status=1):
STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.

Increase the number of iterations (max_iter) or scale the data as shown in:
    https://scikit-learn.org/stable/modules/preprocessing.html
Please also refer to the documentation for alternative solver options:
    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression
  n_iter_i = _check_optimize_result(<jupyter_text>3) Apply / evaluate<jupyter_code>print(logreg.predict(X_train[:10]))
logreg.score(X_train, y_train)
logreg.score(X_test, y_test)
from sklearn.metrics import plot_confusion_matrix
plot_confusion_matrix(logreg, X_test, y_test)<jupyter_output><empty_output><jupyter_text>And again---------<jupyter_code>from sklearn.ensemble import RandomForestClassifier
rf = RandomForestClassifier()
rf.fit(X_train, y_train)
rf.score(X_train, y_train)
rf.score(X_test, y_test)
from sklearn.metrics import plot_confusion_matrix
plot_confusion_matrix(logreg, X_test, y_test)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1248138218647602, 'fraction_numerical': 0.015787905868334822, 'mean_word_length': 6.597285067873303, 'pattern_counts': {'":': 0, '<': 24, '<?xml version=': 0, '>': 24, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '446526', 'license': 'CC0-1.0', 'n_tokens_mistral': 1288, 'n_tokens_neox': 1224, 'n_words': 250, 'path': 'notebooks/Landslide_classification.ipynb', 'repo_name': 'amueller/water_hackweek_2020_machine_learning', 'size': '3357'}