<jupyter_start><jupyter_text>Pre-processing<jupyter_code>df = pd.read_csv("pokemon_revised.csv", encoding='utf-8')

# Drop extra indentifiers (Redundant information)
df = df.drop('japanese_name', 1)
df = df.drop('classfication', 1)
#df = df.drop('pokedex_number', 1)
# Combine dataframes
def append_df(df1, df2):
    for col in df1:
        df2[col] = df1[col]
    return df2

# One-hot encoding on Pokemon abilities
def expand_abilities(df):
    clean_abilities = df['abilities'].apply(lambda x: re.findall('(?<=\')[^,].+?(?=\')', x))
    abilities = np.hstack(clean_abilities.values)
    p_array = np.array(clean_abilities)

    # Building term-document matrix
    # https://docs.scipy.org/doc/scipy/reference/generated/scipy.sparse.csr_matrix.html
    indptr = [0]
    indices = []
    data = []
    vocab = {}

    for pkmn in p_array:
        for abil in pkmn:
            index = vocab.setdefault(abil, len(vocab))
            indices.append(index)
            data.append(1)
        indptr.append(len(indices))
    one_hot = csr_matrix((data, indices, indptr), dtype=int).toarray()
    
    inv_vocab = {v: k for k, v in vocab.items()}
    ordered_abil = [inv_vocab[x] for x in range(len(vocab.keys()))]
    df2 = pd.DataFrame()
    df2[ordered_abil] = pd.DataFrame(one_hot)
    return df2
oha = expand_abilities(df)
df = df.drop('abilities', 1)
df = append_df(oha, df)
df.head()
# One-hot encoding the element typing
def one_hot_types(df):
    primary_type = pd.get_dummies(df['type1'])
    secondary_type = pd.get_dummies(df['type2'])
    both_type = primary_type + secondary_type
    df = df.drop('type1', 1)
    df = df.drop('type2', 1)
    return append_df(both_type, df)
df = one_hot_types(df)
df.head()
# One hot encode the generation number
def expand_gen(df):
    gen_num = pd.get_dummies(df['generation'])
    df = df.drop('generation', 1)
    df = append_df(gen_num, df)
    df.rename(columns = {i: "gen_%s" % str(i) for i in range(1,8)}, inplace = True )
    return df
df = expand_gen(df)
df.head()
# Create new feature: sum of weakness multipliers to convey overall weakness
def create_weakness(df):
    re_against = re.compile("^against_.+$")
    against_ = [x for x in df.columns.values if re_against.match(x)]
    df['weakness'] = df[against_].sum(axis=1)
    return df
df = create_weakness(df)
df.head()
# One hot encode the weakness multipliers
def expand_against(df):
    re_against = re.compile("^against_.+$")
    against_ = [x for x in df.columns.values if re_against.match(x)]
    for item in against_:
        effective = pd.get_dummies(df[item])
        effective.rename(columns = {val: "%s_%s" % (item, val) for val in effective.columns}, inplace = True)
        df = df.drop(item, 1)
        df = append_df(effective, df)
    return df
df = expand_against(df)
df.head()
# Separate the labels
poke_names = df['name']
legendary_labels = df['is_legendary']
df.drop('name', 1, inplace=True)
df.drop('is_legendary', 1, inplace=True)
# We only have male percentages, but that does not convey the Pokemon with no genders
df['percentage_female'] = 100 - df['percentage_male']
df['percentage_male'].fillna(0, inplace = True)
df['percentage_female'].fillna(0, inplace = True)<jupyter_output><empty_output><jupyter_text>K-means ClusteringGroup the Pokemon by power and physical features.Parameters: * n_clusters: How many groups to create* random_state: Random seed<jupyter_code>from sklearn.cluster import KMeans
from sklearn.cluster import SpectralClustering
from sklearn.decomposition import PCA
from mpl_toolkits.mplot3d import Axes3D

stats = ['pokedex_number', 'hp', 'attack', 'defense', 'sp_attack', 'sp_defense', 'speed', 'base_total', 'height_m', 'weight_kg', 'weakness']
pca = PCA(n_components=3)
n_clusters = 25

# Group the pokemon
X = df[stats]
kmeans = KMeans(n_clusters=n_clusters, random_state=5).fit(X[stats[1:]])
pred = kmeans.labels_


# PCA to "flatten" data for visualization
pca.fit(X[stats[1:]]) 
X_pca = pca.transform(X[stats[1:]])
X_pca = X_pca - np.mean(X_pca)
X_pca = X_pca / np.max(X_pca)

fig = plt.figure()
ax = fig.add_subplot(111, projection='3d')
ax.scatter(xs=X_pca[:,0], ys=X_pca[:,1], zs=X_pca[:,2], c=pred)
for i in range(n_clusters):
    print("Group", i, ":", list(poke_names[X[pred == i]['pokedex_number'].values - 1]), "\n")
X2 = X.copy(deep=True)
X2["pokedex_number"]=poke_names

# Example group
X2[pred==6]<jupyter_output><empty_output><jupyter_text>Spectral ClusteringA different method of unsupervised grouping<jupyter_code>from sklearn.cluster import SpectralClustering

pca = PCA(n_components=3)
n_clusters = 25
X = df[stats]
spec_clust = SpectralClustering(n_clusters=n_clusters, affinity='nearest_neighbors', random_state=5).fit(X[stats[1:]])
pred = spec_clust.labels_

pca.fit(X[stats[1:]]) 
X_pca = pca.transform(X[stats[1:]])
X_pca = X_pca - np.mean(X_pca)
X_pca = X_pca / np.max(X_pca)

from mpl_toolkits.mplot3d import Axes3D
fig = plt.figure()
ax = fig.add_subplot(111, projection='3d')
ax.scatter(xs=X_pca[:,0], ys=X_pca[:,1], zs=X_pca[:,2], c=pred)
for i in range(n_clusters):
    print("Group", i, ":", list(poke_names[X[pred == i]['pokedex_number'].values - 1]), "\n")
# Example group
X2[pred==5]
""" 
# Trying random projection/LFA on pokemon types to see if unsupervised learning can learn typings?

from sklearn import random_projection
from sklearn.decomposition import TruncatedSVD

tsvd = TruncatedSVD(n_components=17)
X_sparse_tsvd = tsvd.fit(both_type).transform(both_type)
X_sparse_tsvd.shape

transformer = random_projection.SparseRandomProjection()
#X_new = transformer.fit_transform(both_type)
"""<jupyter_output><empty_output><jupyter_text>BINARY CLASSIFICATION: Trying different methods of predicting if a Pokemon is legendary<jupyter_code>from sklearn.metrics import confusion_matrix
from sklearn.utils.multiclass import unique_labels
from sklearn.model_selection import train_test_split
from sklearn.metrics import accuracy_score

# https://scikit-learn.org/stable/auto_examples/model_selection/plot_confusion_matrix.html
def plot_confusion_matrix(y_true, y_pred, classes,
                          normalize=False,
                          title=None,
                          cmap=plt.cm.Blues):
    """
    This function prints and plots the confusion matrix.
    Normalization can be applied by setting `normalize=True`.
    """
    if not title:
        if normalize:
            title = 'Normalized confusion matrix'
        else:
            title = 'Confusion matrix, without normalization'

    # Compute confusion matrix
    cm = confusion_matrix(y_true, y_pred)
    # Only use the labels that appear in the data
    classes = classes[unique_labels(y_true, y_pred)]
    if normalize:
        cm = cm.astype('float') / cm.sum(axis=1)[:, np.newaxis]
        print("Normalized confusion matrix")
    else:
        print('Confusion matrix, without normalization')

    print(cm)

    fig, ax = plt.subplots()
    im = ax.imshow(cm, interpolation='nearest', cmap=cmap)
    ax.figure.colorbar(im, ax=ax)
    # We want to show all ticks...
    ax.set(xticks=np.arange(cm.shape[1]),
           yticks=np.arange(cm.shape[0]),
           # ... and label them with the respective list entries
           xticklabels=classes, yticklabels=classes,
           title=title,
           ylabel='True label',
           xlabel='Predicted label')

    # Rotate the tick labels and set their alignment.
    plt.setp(ax.get_xticklabels(), rotation=45, ha="right",
             rotation_mode="anchor")

    # Loop over data dimensions and create text annotations.
    fmt = '.2f' if normalize else 'd'
    thresh = cm.max() / 2.
    for i in range(cm.shape[0]):
        for j in range(cm.shape[1]):
            ax.text(j, i, format(cm[i, j], fmt),
                    ha="center", va="center",
                    color="white" if cm[i, j] > thresh else "black")
    fig.tight_layout()
    return ax

def print_misclass(X, y_true, y_pred):
    #match_names = poke_names[X[:,10]-1].values
    miss = X[:,10][y_true != y_pred]
    for i in miss:
        i = int(i)
        if i not in y_true:
            print(poke_names[i-1] + " is not a legendary.")
        else:
            print(poke_names[i-1] + " is a legendary.")<jupyter_output><empty_output><jupyter_text>Standard SVM<jupyter_code>design_matrix = df.values.astype(np.float32)
X_train, X_test, y_train, y_test = train_test_split(
    design_matrix, legendary_labels, test_size = 0.15, random_state = 200)
from sklearn.svm import LinearSVC

X2_train, X_val, y2_train, y_val = train_test_split(
    X_train, y_train, test_size = 0.15, random_state = 0)

# Line search for best C parameter; judge by confusion matrix performance
max_acc = 0
searched_C = -1
for c in np.logspace(-10, 10, 25):
    clf = LinearSVC(random_state=0, tol=1e-4, C = c, max_iter=10000)
    clf.fit(X2_train, y2_train)
    pred_val = clf.predict(X_val)
    cm = confusion_matrix(y_val, pred_val)
    cm = cm.astype('float') / cm.sum(axis=1)[:, np.newaxis]
    val_acc = cm[0][0]+cm[1][1]
    if val_acc > max_acc:
        max_acc=val_acc
        searched_C=c
        plot_confusion_matrix(y_val, pred_val, np.array(['Not Legendary', 'Legendary']), normalize=True)<jupyter_output>Normalized confusion matrix
[[0.99038462 0.00961538]
 [0.18181818 0.81818182]]<jupyter_text>Run model of test data<jupyter_code>X_train, X_test, y_train, y_test = train_test_split(
    design_matrix, legendary_labels, test_size = 0.15, random_state = 200)
X2_train, X_val, y2_train, y_val = train_test_split(
    X_train, y_train, test_size = 0.15, random_state = 0)

clf = LinearSVC(random_state=0, tol=1e-4, C = searched_C, max_iter=50000)
clf.fit(X2_train, y2_train)
pred_test = clf.predict(X_test)
plot_confusion_matrix(y_test, pred_test, np.array(['Not Legendary', 'Legendary']), normalize=True)
print_misclass(X_test, y_test, pred_test)<jupyter_output>Mega-Diancie is not a legendary.
Manaphy is not a legendary.
Silvally is not a legendary.<jupyter_text>RBF Kernelized SVMSeparates data with MVGaussian function<jupyter_code>from sklearn.svm import SVC

design_matrix = df.values.astype(np.float32)
X_train, X_test, y_train, y_test = train_test_split(
    design_matrix, legendary_labels, test_size = 0.15, random_state = 20)
X_train, X_validate, y_train, y_validate = train_test_split(
    X_train, y_train, test_size = 0.15, random_state = 0)

max_acc = 0
searched_C = -1
for c in np.logspace(-0, 10, 11):
    clf = SVC(C = c, kernel = 'rbf', gamma='auto')
    clf.fit(X_train, y_train)
    
    pred_train = clf.predict(X_train)
    plot_confusion_matrix(y_train, pred_train, np.array(['Not Legendary', 'Legendary']), normalize=True)
    
    pred_validate = clf.predict(X_validate)
    cm = confusion_matrix(y_validate, pred_validate)
    cm = cm.astype('float') / cm.sum(axis=1)[:, np.newaxis]    
    validation_acc = cm[0][0]+cm[1][1]
    if validation_acc > max_acc:
        max_acc=validation_acc
        searched_C=c
        plot_confusion_matrix(y_validate, pred_validate, np.array(['Not Legendary', 'Legendary']), normalize=True)
X_train, X_test, y_train, y_test = train_test_split(
    design_matrix, legendary_labels, test_size = 0.15, random_state = 200)
X2_train, X_val, y2_train, y_val = train_test_split(
    X_train, y_train, test_size = 0.15, random_state = 0)

clf = SVC(C = searched_C, kernel = 'rbf', gamma='auto')
clf.fit(X2_train, y2_train)
pred_test = clf.predict(X_test)
plot_confusion_matrix(y_test, pred_test, np.array(['Not Legendary', 'Legendary']), normalize=True)<jupyter_output>Normalized confusion matrix
[[1.         0.        ]
 [0.85714286 0.14285714]]<jupyter_text>Logistic RegressionFits binary classification as probabilities.<jupyter_code>from sklearn import linear_model
from sklearn.linear_model import LogisticRegressionCV
from sklearn.model_selection import train_test_split

design_matrix = df.values.astype(np.float32)
X_train, X_test, y_train, y_test = train_test_split(
    design_matrix, legendary_labels, test_size = 0.4, random_state = 69)
import time

reg = LogisticRegressionCV(Cs=np.logspace(-10, 10, 41), 
                           fit_intercept=True,
                           dual = False,
                          solver='lbfgs',
                           tol=1e-6,
                          max_iter=1e5)

start = time.time()
reg.fit(X_train, y_train)
end = time.time()
print("Fitting took %s ms" % str(end-start))
pred_train = reg.predict(X_train)
plot_confusion_matrix(y_train, pred_train, np.array(['Not Legendary', 'Legendary']), normalize=True)

pred_test = reg.predict(X_test)
plot_confusion_matrix(y_test, pred_test, np.array(['Not Legendary', 'Legendary']), normalize=True)
#print_misclass(X_train, y_train, pred_train)
pred_test = reg.predict(X_train)
#print_misclass(X_test, y_test, pred_test)

match_names = poke_names[X_train[:,10]-1].values
pred_test = reg.predict_proba(X_train)
as_prob = np.around(pred_test, decimals=5)[:,1]
for i in range(len(match_names)):
    print('{:>15}: {:.3%}'.format(match_names[i], as_prob[i]))<jupyter_output>Volbeat: 3.740%
      Lilligant: 1.800%
       Sharpedo: 0.473%
     Gothitelle: 1.556%
      Vileplume: 1.121%
        Honedge: 0.941%
     Kricketune: 0.887%
    Mega Pinsir: 0.971%
      Clawitzer: 0.402%
       Stantler: 0.481%
      Mightyena: 0.646%
        Pancham: 1.056%
          Lotad: 0.238%
        Marowak: 1.287%
        Shaymin: 98.895%
         Combee: 0.592%
        Cubchoo: 1.323%
        Roselia: 0.707%
       Tsareena: 1.737%
          Minun: 0.679%
        Cosmoem: 97.762%
      Kricketot: 0.259%
  Deoxys-Normal: 98.132%
        Dartrix: 1.022%
  Mega Gyarados: 0.163%
       Oranguru: 0.658%
        Cradily: 21.270%
       Genesect: 98.380%
       Porygon2: 2.426%
          Bagon: 3.097%
       Medicham: 1.337%
     Type: Null: 98.447%
         Emolga: 0.773%
        Geodude: 0.201%
      Heliolisk: 1.854%
       Hoothoot: 0.288%
      Excadrill: 1.577%
       Simisage: 1.477%
     Mega Absol: 2.878%
       Beheeyem: 1.547%
        Walrein: 1.215%
        Gl[...]<jupyter_text>Decision Trees<jupyter_code>from sklearn import tree
df_copy = df
design_matrix = df_copy.values.astype(np.float32)
np.random.seed(0)
X_train, X_test, y_train, y_test = train_test_split(
    design_matrix, legendary_labels, test_size = 0.15, random_state = 69)

clf = tree.DecisionTreeClassifier(min_samples_split =5)
clf.fit(X_train, y_train)

pred_train = clf.predict(X_train)
plot_confusion_matrix(y_train, pred_train, np.array(['Not Legendary', 'Legendary']), normalize=False)
pred_test = clf.predict(X_test)
plot_confusion_matrix(y_test, pred_test, np.array(['Not Legendary', 'Legendary']), normalize=False)
print_misclass(X_train, y_train, pred_train)
#X_train[:,10]
print_misclass(X_test, y_test, pred_test)
# Decision Tree created
import graphviz 
dot_data = tree.export_graphviz(clf, out_file=None,
                                feature_names = df_copy.columns,
                               class_names=np.array(['Not legendary', 'legendary']),
                               special_characters=True)

graph = graphviz.Source(dot_data) 
graph<jupyter_output><empty_output><jupyter_text>Random Forest<jupyter_code>from sklearn.ensemble import RandomForestClassifier
#df_copy = df.drop('base_egg_steps', axis=1)
#df_copy = df_copy.drop('capture_rate', axis=1)
#df_copy = df_copy.drop('pokedex_number', axis=1)
design_matrix = df_copy.values.astype(np.float32)
np.random.seed(0)
X_train, X_test, y_train, y_test = train_test_split(
    design_matrix, legendary_labels, test_size = 0.5, random_state = 69)

clf = RandomForestClassifier(n_estimators=5000, 
                             max_depth=2,
                            max_features=100)

clf.fit(X_train, y_train)

pred_train = clf.predict(X_train)
plot_confusion_matrix(y_train, pred_train, np.array(['Not Legendary', 'Legendary']), normalize=True)
pred_test = clf.predict(X_test)
plot_confusion_matrix(y_test, pred_test, np.array(['Not Legendary', 'Legendary']), normalize=True)
from sklearn.tree import export_graphviz
dot_data = export_graphviz(clf.estimators_[np.random.randint(0,5001)], out_file=None,
                                feature_names = df_copy.columns,
                               class_names=np.array(['Not legendary', 'legendary']),
                               special_characters=True)

graph = graphviz.Source(dot_data) 
graph<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12732710736739303, 'fraction_numerical': 0.028871391076115485, 'mean_word_length': 3.793446459918081, 'pattern_counts': {'":': 2, '<': 38, '<?xml version=': 0, '>': 41, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '10', 'dir': 'jupyter-structured-clean-dedup', 'id': '375939', 'license': 'MIT', 'n_tokens_mistral': 6335, 'n_tokens_neox': 5964, 'n_words': 1246, 'path': 'pokemon_ml.ipynb', 'repo_name': 'zhaohanson1/pokemon-ML', 'size': '16383'}