<jupyter_start><jupyter_text>BaseBall Most Valuable Player<jupyter_code>## Import dependencies here

import os
import datetime
import numpy as np
import pandas as pd
from matplotlib import pyplot
from pprint import pprint

# current date/time
now = datetime.datetime.now()<jupyter_output><empty_output><jupyter_text>Data Acquisition<jupyter_code># Read Baseball datasets into individual dataframes

# location of baseball datasets
dataset_path = "data/mvp_baseball_data/core/"

# load Player data into the dataframe
master_df = pd.read_csv(os.path.join(dataset_path, 'Master.csv'),
                        usecols=['playerID','nameFirst','nameLast','bats','throws','debut','finalGame', 'deathYear'])

# load Fielding data into the dataframe
fielding_df = pd.read_csv(os.path.join(dataset_path, 'Fielding.csv'),
                          usecols=['playerID','yearID','stint','teamID','lgID','POS','G','GS','InnOuts','PO','A','E','DP'])

# load Batting data into the dataframe
batting_df = pd.read_csv(os.path.join(dataset_path, 'Batting.csv'))

# load Pitching data into the dataframe
pitching_df = pd.read_csv(os.path.join(dataset_path, 'Pitching.csv'))

# load Awards data into the dataframe
awards_df = pd.read_csv(os.path.join(dataset_path, 'AwardsPlayers.csv'), 
                        usecols=['playerID','awardID','yearID'])

# load All-Star category data into the dataframe
allstar_df = pd.read_csv(os.path.join(dataset_path, 'AllstarFull.csv'),
                         usecols=['playerID','yearID'])

# load Appearances in Baseball match into the dataframe
appearances_df = pd.read_csv(os.path.join(dataset_path, 'Appearances.csv'))<jupyter_output><empty_output><jupyter_text>Data Cleanup & Preparation<jupyter_code># Eliminate players those are not alive any more
master_df = master_df[master_df['deathYear'].isnull()]
master_df = master_df.drop(columns=['deathYear'])
master_df.head()
# batting stats
batting_df.head()
# Initialize dictionaries for player stats and years played
player_stats = {}
years_played = {}

# Create dictionaries for player stats and years played from `batting_df`
for i, row in batting_df.iterrows():
    playerID = row['playerID']
    if playerID in player_stats:
        player_stats[playerID]['G'] = player_stats[playerID]['G'] + row['G']
        player_stats[playerID]['AB'] = player_stats[playerID]['AB'] + row['AB']
        player_stats[playerID]['R'] = player_stats[playerID]['R'] + row['R']
        player_stats[playerID]['H'] = player_stats[playerID]['H'] + row['H']
        player_stats[playerID]['2B'] = player_stats[playerID]['2B'] + row['2B']
        player_stats[playerID]['3B'] = player_stats[playerID]['3B'] + row['3B']
        player_stats[playerID]['HR'] = player_stats[playerID]['HR'] + row['HR']
        player_stats[playerID]['RBI'] = player_stats[playerID]['RBI'] + row['RBI']
        player_stats[playerID]['SB'] = player_stats[playerID]['SB'] + row['SB']
        player_stats[playerID]['BB'] = player_stats[playerID]['BB'] + row['BB']
        player_stats[playerID]['SO'] = player_stats[playerID]['SO'] + row['SO']
        player_stats[playerID]['IBB'] = player_stats[playerID]['IBB'] + row['IBB']
        player_stats[playerID]['HBP'] = player_stats[playerID]['HBP'] + row['HBP']
        player_stats[playerID]['SH'] = player_stats[playerID]['SH'] + row['SH']
        player_stats[playerID]['SF'] = player_stats[playerID]['SF'] + row['SF']
        years_played[playerID].append(row['yearID'])
    else:
        player_stats[playerID] = {}
        player_stats[playerID]['G'] = row['G']
        player_stats[playerID]['AB'] = row['AB']
        player_stats[playerID]['R'] = row['R']
        player_stats[playerID]['H'] = row['H']
        player_stats[playerID]['2B'] = row['2B']
        player_stats[playerID]['3B'] = row['3B']
        player_stats[playerID]['HR'] = row['HR']
        player_stats[playerID]['RBI'] = row['RBI']
        player_stats[playerID]['SB'] = row['SB']
        player_stats[playerID]['BB'] = row['BB']
        player_stats[playerID]['SO'] = row['SO']
        player_stats[playerID]['IBB'] = row['IBB']
        player_stats[playerID]['HBP'] = row['HBP']
        player_stats[playerID]['SH'] = row['SH']
        player_stats[playerID]['SF'] = row['SF']
        years_played[playerID] = []
        years_played[playerID].append(row['yearID'])
# Iterate through `years_played` and add the number of years played to `player_stats`
for k, v in years_played.items():
    player_stats[k]['Years_Played'] = len(list(set(v)))
# Initialize `fielder_list`
fielder_list = []

# Add fielding stats to `player_stats` from `fielding_df`
for i, row in fielding_df.iterrows():
    playerID = row['playerID']
    Gf = row['G']
    GSf = row['GS']
    POf = row['PO']
    Af = row['A']
    Ef = row['E']
    DPf = row['DP']
    if playerID in player_stats and playerID in fielder_list:
        player_stats[playerID]['Gf'] = player_stats[playerID]['Gf'] + Gf
        player_stats[playerID]['GSf'] = player_stats[playerID]['GSf'] + GSf
        player_stats[playerID]['POf'] = player_stats[playerID]['POf'] + POf
        player_stats[playerID]['Af'] = player_stats[playerID]['Af'] + Af
        player_stats[playerID]['Ef'] = player_stats[playerID]['Ef'] + Ef
        player_stats[playerID]['DPf'] = player_stats[playerID]['DPf'] + DPf
    else:
        fielder_list.append(playerID)
        player_stats[playerID]['Gf'] = Gf
        player_stats[playerID]['GSf'] = GSf
        player_stats[playerID]['POf'] = POf
        player_stats[playerID]['Af'] = Af
        player_stats[playerID]['Ef'] = Ef
        player_stats[playerID]['DPf'] = DPf

# Create DataFrames for players who already won Most Valuable Player Award
awards_list = [awards_df[awards_df['awardID'] == 'Most Valuable Player']]

# Initialize lists for each of the above DataFrames
mvp_list = []

# Include each of the above lists in `lists`
lists = [mvp_list]

# Add a count for each award for each player in `player_stats`
for index, v in enumerate(awards_list):
    for i, row in v.iterrows():
        playerID = row['playerID']
        award = row['awardID']
        if playerID in player_stats and playerID in lists[index]:
            player_stats[playerID][award] += 1
        else:
            lists[index].append(playerID)
            player_stats[playerID][award] = 1
# Initialize `allstar_list`
allstar_list = []

# Add a count for each Allstar game appearance for each player in `player_stats`
for i, row in allstar_df.iterrows():
    playerID = row['playerID']
    if playerID in player_stats and playerID in allstar_list:
        player_stats[playerID]['AS_games'] += 1
    else:
        allstar_list.append(playerID)
        player_stats[playerID]['AS_games'] = 1
# Convert `player_stats` into a DataFrame
stats_df = pd.DataFrame.from_dict(player_stats, orient='index')

# player stats
stats_df
# Add a column for playerID from the `stats_df` index
stats_df['playerID'] = stats_df.index

# Join `stats_df` and `master_df`
master_df = master_df.join(stats_df,on='playerID',how='inner', rsuffix='mstr')

# Inspect first rows of `master_df`
master_df.head()
# Initialize a dictionary
pos_dict = {}

# Iterate through `appearances_df`
# Add a count for the number of appearances for each player at each position
# Also add a count for the number of games played for each player in each era.
for i, row in appearances_df.iterrows():
    ID = row['playerID']
    year = row['yearID']
    if ID in pos_dict:
        pos_dict[ID]['G_all'] = pos_dict[ID]['G_all'] + row['G_all']
        pos_dict[ID]['G_p'] = pos_dict[ID]['G_p'] + row['G_p']
        pos_dict[ID]['G_c'] = pos_dict[ID]['G_c'] + row['G_c']
        pos_dict[ID]['G_1b'] = pos_dict[ID]['G_1b'] + row['G_1b']
        pos_dict[ID]['G_2b'] = pos_dict[ID]['G_2b'] + row['G_2b']
        pos_dict[ID]['G_3b'] = pos_dict[ID]['G_3b'] + row['G_3b']
        pos_dict[ID]['G_ss'] = pos_dict[ID]['G_ss'] + row['G_ss']
        pos_dict[ID]['G_lf'] = pos_dict[ID]['G_lf'] + row['G_lf']
        pos_dict[ID]['G_cf'] = pos_dict[ID]['G_cf'] + row['G_cf']
        pos_dict[ID]['G_rf'] = pos_dict[ID]['G_rf'] + row['G_rf']
        pos_dict[ID]['G_of'] = pos_dict[ID]['G_of'] + row['G_of']
        pos_dict[ID]['G_dh'] = pos_dict[ID]['G_dh'] + row['G_dh']
    else:
        pos_dict[ID] = {}
        pos_dict[ID]['G_all'] = row['G_all']
        pos_dict[ID]['G_p'] = row['G_p']
        pos_dict[ID]['G_c'] = row['G_c']
        pos_dict[ID]['G_1b'] = row['G_1b']
        pos_dict[ID]['G_2b'] = row['G_2b']
        pos_dict[ID]['G_3b'] = row['G_3b']
        pos_dict[ID]['G_ss'] = row['G_ss']
        pos_dict[ID]['G_lf'] = row['G_lf']
        pos_dict[ID]['G_cf'] = row['G_cf']
        pos_dict[ID]['G_rf'] = row['G_rf']
        pos_dict[ID]['G_of'] = row['G_of']
        pos_dict[ID]['G_dh'] = row['G_dh']
# Convert the `pos_dict` to a DataFrame
pos_df = pd.DataFrame.from_dict(pos_dict, orient='index')

# players played in various positions
pos_df.head()
# Create a list from the columns of `pos_df`
pos_col_list = pos_df.columns.tolist()

# Remove the string 'G_all'
pos_col_list.remove('G_all')

# Loop through the list and divide each column by the players total games played
for col in pos_col_list:
    column = col + '_percent'
    pos_df[column] = pos_df[col] / pos_df['G_all']

# Print out the first rows of `pos_df`    
pos_df.head()
# Filter `pos_df` to eliminate players who played 10% or more of their games as Pitchers or Catchers
pos_df = pos_df[(pos_df['G_p_percent'] < 0.1) & (pos_df['G_c_percent'] < 0.1)]

# Get info on `pos_df`
pos_df.head()
# Join `pos_df` and `master_df`
master_df = master_df.join(pos_df,on='playerID',how='right')

# master_df at this stage after the cleanup and data preparation
master_df.head()
# Create a function to convert the bats and throws colums to numeric
def bats_throws(col):
    if col == "R":
        return 1
    else:
        return 0

# Use the `apply()` method to create numeric columns from the bats and throws columns
master_df['bats_R'] = master_df['bats'].apply(bats_throws)
master_df['throws_R'] = master_df['throws'].apply(bats_throws)

# Print out the first rows of `master_df`
master_df.head()
# Import datetime 
from datetime import datetime

# Convert the `debut` column to datetime
master_df['debut'] =  pd.to_datetime(master_df['debut'])
# Convert the `finalGame` column to datetime
master_df['finalGame'] = pd.to_datetime(master_df['finalGame'])

# Create new columns for debutYear and finalYear
master_df['debutYear'] = pd.to_numeric(master_df['debut'].dt.strftime('%Y'), errors='coerce')
master_df['finalYear'] = pd.to_numeric(master_df['finalGame'].dt.strftime('%Y'), errors='coerce')

# Return the first rows of `master_df`
master_df.head()
# Eliminating unnecessary columns
df = master_df.drop(['IBB', 'bats', 'throws', 'GSf', 'POf','Gf', 'playerIDmstr'], axis=1)

# master_df with only the required columns
df.head()
# Fill null values in numeric columns with 0
fill_cols = ['AS_games', 'Most Valuable Player', 'HBP', 'SB', 'SF', 'SH', 'RBI', 'SO', 'G_dh_percent', 'G_dh', 'Af', 'DPf', 'Ef']

for col in fill_cols:
    df[col] = df[col].fillna(0)
# Create Batting Average (`AVE`) column
df['AVE'] = df['H'] / df['AB']

# Create On Base Percent (`OBP`) column
plate_appearances = (df['AB'] + df['BB'] + df['SF'] + df['SH'] + df['HBP'])
df['OBP'] = (df['H'] + df['BB'] + df['HBP']) / plate_appearances

# Create Slugging Percent (`Slug_Percent`) column
single = ((df['H'] - df['2B']) - df['3B']) - df['HR']
df['Slug_Percent'] = ((df['HR'] * 4) + (df['3B'] * 3) + (df['2B'] * 2) + single) / df['AB']

# Create On Base plus Slugging Percent (`OPS`) column
hr = df['HR'] * 4
triple = df['3B'] * 3
double = df['2B'] * 2
df['OPS'] = df['OBP'] + df['Slug_Percent']

# Double check the `df` columns 
print(df.columns)
df.head()
# Make a function that will create a new column to honor <NAME>, the first African American Major league Baseball Player
def first_aap(col):
    if col == 'robinja02':
        return 1
    else:
        return 0
    
# Apply `first_aap` to `df['playerID']`    
df['first_aap'] = df['playerID'].apply(first_aap)

# Retrieve `df` columns
print(df.columns)
# Create training data
# Players who already won the 'Most Valuable Player' award
df_hitters = df[df['Most Valuable Player'] > 0]
df_hitters['Most Valuable Player'].count()
# create test data
# Players who are eligible for 'Most Valuable Player' award
# df_eligible = df[df['Most Valuable Player'] <= 0]
df_eligible = df
df_eligible['Most Valuable Player'].count()
# Select columns to use for models, and identification columns
num_cols_hitters = ['playerID', 'nameFirst', 'nameLast', 'debut', 'finalGame', 'G', 'AB',
       'R', 'H', '2B', '3B', 'HR', 'RBI', 'SB', 'BB', 'SO', 'HBP', 'SH', 'SF',
       'Years_Played', 'Af', 'Ef', 'DPf', 'Most Valuable Player', 'AS_games',
       'G_all', 'G_p', 'G_c', 'G_1b', 'G_2b', 'G_3b', 'G_ss', 'G_lf', 'G_cf',
       'G_rf', 'G_of', 'G_dh', 'G_p_percent', 'G_c_percent', 'G_1b_percent',
       'G_2b_percent', 'G_3b_percent', 'G_ss_percent', 'G_lf_percent',
       'G_cf_percent', 'G_rf_percent', 'G_of_percent', 'G_dh_percent',
       'bats_R', 'throws_R', 'debutYear', 'finalYear', 'AVE', 'OBP',
       'Slug_Percent', 'OPS', 'first_aap']

# Create a new DataFrame (`data`) from the `df_hitters` using the columns above
data = df_hitters[num_cols_hitters]

data.head()
import plotly as py
import plotly.graph_objs as go
from plotly import tools

py.offline.init_notebook_mode(connected=True)

trace1 = go.Histogram(x=data['H'], name ='Distribution of Hits')
trace2 = go.Histogram(x=data['HR'], name='Distribution of Home Runs')
trace3 = go.Histogram(x=data['AVE'], name='Distribution of Batting Average')
trace4 = go.Histogram(x=data['G'], name='Distribution of Total Games')

fig = tools.make_subplots(rows=2, cols=2)

fig.append_trace(trace1, 1, 1)
fig.append_trace(trace2, 1, 2)
fig.append_trace(trace3, 2, 1)
fig.append_trace(trace4, 2, 2)

fig['layout'].update(height=800, width=800, title='Distribution of various stats')

py.offline.iplot(fig)
# Create `target` Series
target = data['Most Valuable Player']

# Create `features` DataFrame
features = data.drop(['playerID', 'nameFirst', 'nameLast', 'debut', 'debutYear', 'finalGame', 'finalGame', 'finalYear', 'Most Valuable Player'], axis=1)
target.count()
# Import RandomForestClassifier from sklearn
from sklearn.cross_validation import cross_val_predict, KFold
from sklearn.ensemble import RandomForestClassifier

# Create penalty dictionary
penalty = {
    0: 100,
    1: 1
}

# Create Random Forest model
rf = RandomForestClassifier(random_state=1,n_estimators=10, max_depth=1, min_samples_leaf=1, class_weight='balanced')

# Create an instance of the KFold class
kf = KFold(features.shape[0], random_state=1)

# Create predictions using cross validation
predictions_rf = cross_val_predict(rf, features, target, cv=kf)

# Convert predictions to NumPy array
np_predictions_rf = np.asarray(predictions_rf)
# Create a new DataFrame from `df_eligible` using `num_col_hitters`
new_data = df_eligible[num_cols_hitters]

new_data = new_data.fillna(0)

# Create a new features DataFrame
new_features = new_data.drop(['playerID', 'nameFirst', 'nameLast', 'debut', 'debutYear', 'finalGame', 'finalGame', 'finalYear', 'Most Valuable Player'], axis=1)
# Fit the Random Forest model
rf.fit(features, target)

# Estimate probabilities of Hall of Fame induction
probabilities = rf.predict_proba(new_features)
# Create dateframe for top most active valuable players
active_mvp_df = pd.DataFrame(columns=df_eligible.columns.values.tolist())

# Convert predictions to a DataFrame
mvp_predictions = pd.DataFrame(probabilities[:,1])

# Sort the DataFrame (descending)
mvp_predictions = mvp_predictions.sort_values(0, ascending=False)

mvp_predictions['Probability'] = mvp_predictions[0]

# Print 10 highest probability 'Most Valuable Players'

import datetime
now = datetime.datetime.now()
counter = 0

# Sort the predictions based on the probability score
# The higher probability score a player gets, the more chances to win 'Most Valuable Player' award
for i, row in mvp_predictions.sort_values(by='Probability', ascending=False).iterrows():

    # probability score of player to win 'Most Valuable Player' Award
    prob = str(row['Probability'])
 
    if now.year - int(df_eligible.iloc[i]['finalYear']) < 10:
        counter += 1
        active_mvp_df = active_mvp_df.append(df_eligible.iloc[i])

    if counter >= 10:
        # find the top 10 most active valuable players and then break
        break

# drop if there are any columns that may not be necessary for further analysis
active_mvp_df = active_mvp_df.drop(columns=['debut', 'finalGame', 'debutYear', 'finalYear'])
  
# top 10 active players with more chances of winning 'Most Valuable Player' Award
active_mvp_df
# Create a trace
trace = go.Scatter(
   x = data['HR'],
   y = data['AVE'],
   name = 'Most Valuable Player',
   mode = 'markers',
   marker = dict(
       size = 10,
       color = 'rgb(255, 0, 0)',
       line = dict(
           width = 2,
       )
   ))

trace1 = go.Scatter(
   x = active_mvp_df['HR'],
   y = active_mvp_df['AVE'],
   mode = 'markers',
   name = 'Potential Most Valuable Player',    
   marker = dict(
       size = 10,
       color = 'rgb(0, 0, 255)',
       line = dict(
           width = 2,
       )
   ))

layout = dict(title = 'Career Home Runs vs. Career Batting Average',
             yaxis = dict(zeroline = False,title= 'Career Batting Average'),
             xaxis = dict(zeroline = False,title= 'Career Home Runs')
            )

fig = dict(data=[trace, trace1], layout=layout)

# Plot and embed in ipython notebook!
py.offline.iplot(fig)
# Create a trace
trace = go.Scatter(
   x = data['H'],
   y = data['AVE'],
   name = 'Most Valuable Player',
   mode = 'markers',
   marker = dict(
       size = 10,
       color = 'rgb(255, 0, 0)',
       line = dict(
           width = 2,
       )
   ))

trace1 = go.Scatter(
   x = active_mvp_df['H'],
   y = active_mvp_df['AVE'],
   mode = 'markers',
   name = 'Potential Most Valuable Player',    
   marker = dict(
       size = 10,
       color = 'rgb(0, 0, 255)',
       line = dict(
           width = 2,
       )
   ))

layout = dict(title = 'Career Hits vs. Career Batting Average',
             yaxis = dict(zeroline = False,title= 'Career Batting Average'),
             xaxis = dict(zeroline = False,title= 'Career Hits')
            )

fig = dict(data=[trace, trace1], layout=layout)

# Plot and embed in ipython notebook!
py.offline.iplot(fig)
# Conclusion : The higher probably score a player gets, 
#              the better chances of winning 'Most Valuable Player' Award<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1541795997003104, 'fraction_numerical': 0.008402012201648292, 'mean_word_length': 4.230058774139379, 'pattern_counts': {'":': 1, '<': 18, '<?xml version=': 0, '>': 16, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '152018', 'license': 'MIT', 'n_tokens_mistral': 6612, 'n_tokens_neox': 6402, 'n_words': 1577, 'path': 'notebooks/BaseBall_MostValuablePlayer-PredictiveAnalysis.ipynb', 'repo_name': 'kchhajed1/baseball_predictions', 'size': '18695'}