<jupyter_start><jupyter_text>Load in the pre-pitcher preprocessed_df<jupyter_code>#load in pickled preprocessed df

df1 = pd.read_pickle("/content/RC-v1.1--Data-Cleaning-and-Feature-Engineering/data/pre_pitcher_pre_processed_df_part1.pkl", compression='zip')
df2 = pd.read_pickle("/content/RC-v1.1--Data-Cleaning-and-Feature-Engineering/data/pre_pitcher_pre_processed_df_part2.pkl", compression='zip')

df = pd.concat([df1, df2])
#clear some memory:
del df1
del df2
gc.collect()
#make list of pitchers (for filling in Nans in batting stat later for pitcher slot in batting order)

def make_pitcher_list(pre_processed_df):
    #make list of all the pitcher ids in the preprocessed_df
    pitcher_list = df['pitcher'].unique().tolist()
    
    #on rare occassions in extra-inning games, a position player is called in to pitch: 
    #to avoid categorizing him as a pitcher we'll only count pitchers with over 100 pitches:
    for pitcher in pitcher_list:
        if len(df[df['pitcher'] == pitcher]) < 100:
            pitcher_list.remove(pitcher)
    return pitcher_list

pitcher_list = make_pitcher_list(df)<jupyter_output><empty_output><jupyter_text>Choose the pitcher to model<jupyter_code>#df.groupby('pitcher').count().sort_values('pitch_type', ascending=False).head(10)
#dictionary mapping top 10 pitchers with most pitches name with pitcher id

pitcher_id_dict = {'<NAME>': 434378.0, '<NAME>': 545333.0, 
                   '<NAME>': 605400.0, '<NAME>': 458681.0, '<NAME>':
                   543037.0, '<NAME>':594798.0, '<NAME>':425844.0,
                   '<NAME>': 571578.0, '<NAME>': 453286.0,
                   '<NAME>': 502043.0}

#make pitcher_df subset where pitcher == picther_id
pitcher_id = pitcher_id_dict['<NAME>']
# pitcher_id = pitcher_id_dict['<NAME>']
# pitcher_id = pitcher_id_dict['<NAME>']
# pitcher_id = pitcher_id_dict['<NAME>']
# pitcher_id = pitcher_id_dict['<NAME>']
# pitcher_id = pitcher_id_dict['<NAME>']
# pitcher_id = pitcher_id_dict['<NAME>']
# pitcher_id = pitcher_id_dict['<NAME>']
# pitcher_id = pitcher_id_dict['<NAME>']
# pitcher_id = pitcher_id_dict['<NAME>']

pitcher_df = df[df['pitcher'] == pitcher_id]
pitcher_df.head()<jupyter_output><empty_output><jupyter_text>Add the pitcher prior tendencies (Pitcher Scouting Report)<jupyter_code>%%time
#read in 2017 pre_procesed step 1 pickle and filter for pitcher_id
df17 = pd.read_pickle('/content/RC-v1.1--Data-Cleaning-and-Feature-Engineering/data/preprocessed_2017.pkl', compression = 'zip')
pitcher_df17 = df17[df17['pitcher'] == pitcher_id]
def get_left_right_pitch_tendencies(pitcher_df):
    #split the df into left hand and right handed batters
    left = pitcher_df[pitcher_df['stand'] == 'L'].copy()
    right = pitcher_df[pitcher_df['stand'] == 'R'].copy()
    
    #assign the normalized value counts for this pitchers pitch types to a dictionary
    overall_left = left['pitch_cat'].value_counts(normalize=True).to_dict()
    overall_right = right['pitch_cat'].value_counts(normalize=True).to_dict()
    
    #initialize empty dict for count categories tendencies
    by_count_left = {}
    by_count_right = {}
    
    #loop over each count category and get the pitchers tendencies and add to the dict
    for cat in pitcher_df['count_cat'].unique().tolist():
        left_subset = left[left['count_cat'] == cat]
        right_subset = right[right['count_cat'] == cat]
        
        by_count_left[cat] = left_subset['pitch_cat'].value_counts(normalize=True).to_dict()
        by_count_right[cat] = right_subset['pitch_cat'].value_counts(normalize=True).to_dict()
        
    return overall_left, overall_right, by_count_left, by_count_right

def make_tendency_features(pitcher_df, overall_left, overall_right, by_count_left, by_count_right):
    #helper functions to vectorize w/ df.apply():
    def get_overall_left_perc(x):
        return overall_left[x] * 100
    def get_overall_right_perc(x):
        return overall_right[x] * 100
    def get_by_count_left_perc(x):
        try:
            return by_count_left[x][pitch_type] * 100
        except KeyError:
            return 0
    def get_by_count_right_perc(x):
        try:
            return by_count_right[x][pitch_type] * 100
        except KeyError:
            return 0
    
    left = pitcher_df[pitcher_df['stand'] == 'L'].copy()
    right = pitcher_df[pitcher_df['stand'] == 'R'].copy()
    
    pitch_types_left = overall_left.keys()
    pitch_types_right = overall_right.keys()
    
    #Left
    for pitch_type in pitch_types_left:
        overall_feature = 'overall_' + pitch_type + '_perc'
        count_cat_feature = 'count_cat_' + pitch_type + '_perc'
      
        left[overall_feature] = pitch_type
        left[overall_feature] = left[overall_feature].apply(get_overall_left_perc)
        left[count_cat_feature] = left['count_cat'].apply(get_by_count_left_perc)
    #Right
    for pitch_type in pitch_types_right:
        overall_feature = 'overall_' + pitch_type + '_perc'
        count_cat_feature = 'count_cat_' + pitch_type + '_perc'
      
        right[overall_feature] = pitch_type
        right[overall_feature] = right[overall_feature].apply(get_overall_right_perc)
        right[count_cat_feature] = right['count_cat'].apply(get_by_count_right_perc)
        
    
    return pd.concat([left,right], sort=False).sort_values(by=['game_date', 'game_pk', 'at_bat_number', 'pitch_number'])

def add_pitcher_scouting_report(pitcher_df, pitcher_df17, start_dates, end_dates):
    df = pd.concat([pitcher_df, pitcher_df17], sort=False)
    
    #initialize empty list to store dfs (concat them together later)
    df_list = []
    
    #iterate over each period
    for i in range(len(start_dates)):
      
        #make the prior and current dfs:
        prior_df = df[df['game_date'] < start_dates[i]]
        current_df = df[(df['game_date'] >= start_dates[i]) & (df['game_date'] <= end_dates[i])].copy()
        
        #get the pitch tendencies from prior:
        overall_left, overall_right, by_count_left, by_count_right = get_left_right_pitch_tendencies(prior_df)
        
        #make the pitch tendencies features on current:
        current_df = make_tendency_features(current_df, overall_left, overall_right, by_count_left, by_count_right)
        
        #append the df to the list
        df_list.append(current_df)
    
    df = pd.concat(df_list, sort=False)
    return df

start_dates = ['2018-03-29', '2018-05-01', '2018-06-01', '2018-07-01', '2018-08-01', 
               '2018-09-01', '2019-03-28', '2019-05-01', '2019-06-01', '2019-07-01', 
               '2019-08-01']

end_dates =  ['2018-04-30', '2018-05-31', '2018-06-30', '2018-07-31', '2018-08-31', 
              '2018-10-01', '2019-04-30', '2019-05-31', '2019-06-30', '2019-07-31', 
              '2019-08-31']
%%time
pitcher_df = add_pitcher_scouting_report(pitcher_df, pitcher_df17, start_dates, end_dates)
pitcher_df.head()<jupyter_output><empty_output><jupyter_text>Add Game Features: Batting Order, Game pitch_count, & Trailing Pitch Features<jupyter_code>def make_game_batting_order(game_df):
    game_df = game_df.sort_values(by=['at_bat_number', 'pitch_number'])
    all_batters = game_df['batter'].unique().tolist()
    #re-set the at_bat_number for the game to be sequential starting at 1
    at_bat_keys = game_df['at_bat_number'].unique().tolist()
    at_bat_values = range(1, len(at_bat_keys)+1)
    at_bat_map = dict(zip(at_bat_keys, at_bat_values))
    game_df['at_bat_number'] = game_df['at_bat_number'].replace(at_bat_map)
    
    #get the first 9 batter ids
    first_9_batter_subset = game_df[game_df['at_bat_number'] < 10]
    first_9_batters = first_9_batter_subset['batter'].unique().tolist()
    
    #map the batter id to batting order position 1-9
    batting_order_map = dict(zip(first_9_batters, range(1,10)))
    
    #for anyone else who bats later in the game, assign 'PH' (pinch hitter) to their batting order slot
    other_batters = list(set(all_batters) - set(first_9_batters))
    if len(other_batters) > 0:
        for batter in other_batters:
            batting_order_map[batter] = 'PH'
    try:
        game_df['batting_order_slot'] = game_df['batter'].apply(lambda x: batting_order_map[x])    
    except KeyError:
        game_df = None
        return game_df
    
    game_df['pitcher_AB'] = game_df['batter'].apply(lambda x: True if x in pitcher_list else False)
    game_df['batting_order_slot'] = game_df['batting_order_slot'].where(game_df['pitcher_AB'] == False, other='pitcher')
    return game_df

def get_pitch_tendencies(pitcher_df):
    #assign the normalized value counts for this pitchers pitch types to a dictionary
    pitcher_tendencies_overall = pitcher_df['pitch_cat'].value_counts(normalize=True).to_dict()

    #initialize empty dict for count categories tendencies
    pitcher_tendencies_by_count = {}
    
    #loop over each count category and get the pitchers tendencies and add to the dict
    for cat in pitcher_df['count_cat'].unique().tolist():
        subset = pitcher_df[pitcher_df['count_cat'] == cat]
        pitcher_tendencies_by_count[cat] = subset['pitch_cat'].value_counts(normalize=True).to_dict()
    return pitcher_tendencies_overall, pitcher_tendencies_by_count

def make_game_pitchcount_and_trailing_pitch_features_and_batting_order(pitcher_df, pitcher_list):
    df = pitcher_df.copy()
    all_games = []
    
    print('#pitches in df before: ' + str(len(df)))
    
    pitcher_tendencies_overall, pitcher_tendencies_by_count = get_pitch_tendencies(df)
    games = df['game_pk'].unique().tolist()
    
    for game in games:
        
        #take the first game and make the pitch count feature
        game_df = df[df['game_pk'] == game].copy()
        game_df['pitch_count'] = range(1, game_df.shape[0] + 1)
    
        #make the L1_pitch type feature:
        game_df['L1_pitch_type'] = game_df['pitch_cat'].shift(periods=1)
        game_df['L1_pitch_result'] = game_df['type'].shift(periods=1)
        game_df['L1_pitch_result'] = game_df['L1_pitch_result'].replace({np.nan:'first pitch'})
        game_df['L1_pitch_zone'] = game_df['zone'].shift(periods=1)
        game_df['L1_ball_high'] = game_df['ball_high'].shift(periods=1)
        game_df['L1_ball_low'] = game_df['ball_low'].shift(periods=1)
        game_df['L1_ball_left'] = game_df['ball_left'].shift(periods=1)
        game_df['L1_ball_right'] = game_df['ball_right'].shift(periods=1)
        
        game_df[['L1_pitch_zone', 'L1_ball_high', 'L1_ball_low', 'L1_ball_left', 'L1_ball_right']] = game_df[['L1_pitch_zone', 'L1_ball_high', 'L1_ball_low', 'L1_ball_left', 'L1_ball_right']].fillna(-1)
        #game_df['L1_pitch_zone'] = game_df['L1_pitch_zone'].fillna(-1)

        #overall strike % (to fill in for first 5 pitches L5_strike_perc)
        overall_strike_perc = df['type'].value_counts(normalize=True)['S'] * 100

        #make the trailing 5 pitches:
        for index, row in game_df.iterrows():
            #fill NaNs for L1_pitch using same method as when pitch_type was missing
            if row['pitch_count'] == 1:
                random_pitch = random.choices(population=list(pitcher_tendencies_overall.keys()), 
                                   weights=list(pitcher_tendencies_overall.values()), 
                                   k=1)[0]
                game_df.at[index, 'L1_pitch_type'] = random_pitch

            #for the first 5 rows, use overall pitcher tendencies    
            if row['pitch_count'] < 6:
                #fill with overall tendencies
                for pitch in list(pitcher_tendencies_overall.keys()):
                    feature = 'L5_' + pitch + '_perc'
                    game_df.at[index, feature] = pitcher_tendencies_overall[pitch] * 100
                    feature = 'L15_' + pitch + '_perc'
                    game_df.at[index, feature] = pitcher_tendencies_overall[pitch] * 100
                #strike %
                game_df.at[index, 'L5_strike_perc'] = overall_strike_perc
                game_df.at[index, 'L15_strike_perc'] = overall_strike_perc

            else:
                current_pitch = game_df.at[index, 'pitch_count']
                #make a subset of the prev 5 pitches
                subset = game_df[(game_df['pitch_count'] > current_pitch - 6) & (game_df['pitch_count'] < current_pitch)]
                #grab the value count percentages for the last 5 pitches
                subset_percentages = subset['pitch_cat'].value_counts(normalize=True).to_dict()
                try:
                    L5_strike_perc = subset['type'].value_counts(normalize=True)['S'] * 100
                except KeyError:
                    L5_strike_perc = 0

                game_df.at[index, 'L5_strike_perc'] = L5_strike_perc

                #iterate over all possible pitch types this pitcher throws:
                for pitch in list(pitcher_tendencies_overall.keys()):
                    feature = 'L5_' + pitch + '_perc'
                    #if he has thrown that pitch type in last 5
                    try:
                        game_df.at[index, feature] = subset_percentages[pitch] * 100
                    #except for when he hasnt thrown that type in last 5
                    except:
                        game_df.at[index, feature] = 0


                if row['pitch_count'] < 16:
                    #make a subset of the prev 15 pitches
                    subset = game_df[(game_df['pitch_count'] < current_pitch)]
                    #grab the value count percentages for the last 15 pitches
                    subset_percentages = subset['pitch_cat'].value_counts(normalize=True).to_dict()
                    try:
                        L15_strike_perc = subset['type'].value_counts(normalize=True)['S'] * 100
                    except KeyError:
                        L15_strike_perc = 0

                    game_df.at[index, 'L15_strike_perc'] = L15_strike_perc

                    #iterate over all possible pitch types this pitcher throws:
                    for pitch in list(pitcher_tendencies_overall.keys()):
                        feature = 'L15_' + pitch + '_perc'
                        #if he has thrown that pitch type in last 15
                        try:
                            game_df.at[index, feature] = subset_percentages[pitch] * 100
                        #except for when he hasnt thrown that type in last 5
                        except:
                            game_df.at[index, feature] = 0
                else:
                    #make a subset of the prev 15 pitches
                    subset = game_df[(game_df['pitch_count'] > current_pitch - 16) & (game_df['pitch_count'] < current_pitch)]
                    #grab the value count percentages for the last 5 pitches
                    subset_percentages = subset['pitch_cat'].value_counts(normalize=True).to_dict()
                    try:
                        L15_strike_perc = subset['type'].value_counts(normalize=True)['S'] * 100
                    except KeyError:
                        L15_strike_perc = 0

                    game_df.at[index, 'L15_strike_perc'] = L15_strike_perc

                    #iterate over all possible pitch types this pitcher throws:
                    for pitch in list(pitcher_tendencies_overall.keys()):
                        feature = 'L15_' + pitch + '_perc'
                        #if he has thrown that pitch type in last 5
                        try:
                            game_df.at[index, feature] = subset_percentages[pitch] * 100
                        #except for when he hasnt thrown that type in last 5
                        except:
                            game_df.at[index, feature] = 0
                            
        #apply the battting order features to the game:                
        game_df = make_game_batting_order(game_df)
    
        all_games.append(game_df)
    
    new_df = pd.concat(all_games).sort_values(by=['game_date', 'game_pk', 'at_bat_number', 'pitch_number'])
    
    print('# pitches in df after: ' + str(len(new_df)))
    
    return new_df

def make_prev_ab_walk_basehit_run_and_homerun_features(pitcher_df):
    
    all_games = []
    #iterate over each game
    for game in pitcher_df['game_pk'].unique():
        #make subset df for that game
        game_df = pitcher_df[pitcher_df['game_pk'] == game].copy()
        #initialize columns to False:
        game_df['prev_ab_run_scored'] = False
        game_df['prev_ab_homerun'] = False
        game_df['prev_ab_walk'] = False
        game_df['prev_ab_basehit'] = False
        game_df['prev_ab_strikeout'] = False
        
        #this gets the 
        at_bats = game_df['at_bat_number'].sort_values().unique()
        
        #initialize empty dicts
        run_scored = []
        homeruns = []
        walks = []
        basehits = []
        strikeouts = []
        
        walks = ['walk', 'hit_by_pitch']
        basehits = ['single', 'double', 'triple', 'home_run']
        
        #starting w/ 2nd AB, iterate thru to the end of the at_bats:
        for ab in at_bats[2:]:
            #get the index for the last pitch of the prev AB
            prev_ab_last_pitch_index = game_df[game_df['at_bat_number'] == ab-1]['pitch_number'].index.max()
            #check if the last pitch resulted in a walk or hit by pitch:
            if game_df.loc[prev_ab_last_pitch_index]['events'] in walks:
                #if so, add an entry
                walks.append(ab)
            #check if last pitch gave up a basehit:
            elif game_df.loc[prev_ab_last_pitch_index]['events'] in basehits:
                basehits.append(ab)
            elif game_df.loc[prev_ab_last_pitch_index]['events'] == 'strikeout':
                strikeouts.append(ab)
            
            #to check if prev AB resulted in a run scoring: compare score before and after the AB
            prev_score = game_df[game_df['at_bat_number'] == ab-1]['bat_score'].values[0]
            current_score = game_df[game_df['at_bat_number'] == ab]['bat_score'].values[0]
            
            if current_score > prev_score:
                run_scored.append(ab)
                 
                #check if last AB gave up a homerun:
                if game_df.loc[prev_ab_last_pitch_index]['events'] == 'home_run':
                    homeruns.append(ab)
                    
        #iterate over each at_bat, and add the features to the df where appropriate
        for ab in at_bats:
            idx = game_df[game_df['at_bat_number'] == ab].index
            if ab in walks:
                game_df.at[idx, 'prev_ab_walk'] = True
            elif ab in basehits:
                game_df.at[idx, 'prev_ab_basehit'] = True
            elif ab in strikeouts:
                game_df.at[idx, 'prev_ab_strikeout'] = True
            if ab in run_scored:
                game_df.at[idx, 'prev_ab_run_scored'] = True
                if ab in homeruns:
                    game_df.at[idx, 'prev_ab_homerun'] = True
        all_games.append(game_df)
        
    return pd.concat(all_games).sort_values(by=['game_date', 'game_pk', 'pitch_count'])
%%time
pitcher_df = make_game_pitchcount_and_trailing_pitch_features_and_batting_order(pitcher_df, pitcher_list)
%%time
pitcher_df = make_prev_ab_walk_basehit_run_and_homerun_features(pitcher_df)
#pitcher_df[['game_pk','at_bat_number','bat_score','events', 'prev_ab_run_scored', 'prev_ab_homerun', 'prev_ab_walk', 'prev_ab_basehit', 'prev_ab_strikeout']].head(10)
cols = ['pitch_count', 'pitch_type', 'type', 'batting_order_slot']	

for col in pitcher_df.columns:
    if col[:2] == 'L1':
        cols.append(col)
    if col[-4:] == 'perc':
        if col[:2]=='L5':
            cols.append(col)
        if col[:3]=='L15':
            cols.append(col)
        
pitcher_df[cols].head(20)
pitcher_df[cols].tail(20)<jupyter_output><empty_output><jupyter_text>Fill Batting Scouting Report NaNs Load in the batting_order_slot map pickle<jupyter_code>batting_order_slot_map = pickle.load(open("/content/RC-v1.1--Data-Cleaning-and-Feature-Engineering/data/batting_order_slot_map.pkl", "rb"))
#batting_order_slot_map
batter_cols = ['fastball_perc_faced','fastball_chase_perc','fastball_bip_swung_perc', 'fastball_taken_strike_perc',
               'fastball_est_woba', 'fastball_babip', 'fastball_iso_value', 'breaking_perc_faced', 'breaking_chase_perc',
               'breaking_bip_swung_perc', 'breaking_taken_strike_perc', 'breaking_est_woba', 'breaking_babip', 
               'breaking_iso_value', 'offspeed_perc_faced', 'offspeed_chase_perc', 'offspeed_bip_swung_perc',
               'offspeed_taken_strike_perc', 'offspeed_est_woba', 'offspeed_babip', 'offspeed_iso_value',
               'pitchout_perc_faced']

def fill_batting_nans(pitcher_df, batting_order_slot_map):
    df = pitcher_df.copy()
    for slot in df['batting_order_slot'].unique().tolist():
        subset = df[df['batting_order_slot'] == slot].copy()
        df = df.drop(subset.index)
        for col in batter_cols:
            subset[col] = subset[col].fillna(batting_order_slot_map[slot][col])
        df = pd.concat([df, subset])
        print('finished w/ slot: ' + str(slot))
    df = df.sort_values(by=['game_date', 'game_pk', 'pitch_count'])
    return df
%%time
pitcher_df = fill_batting_nans(pitcher_df, batting_order_slot_map)<jupyter_output>finished w/ slot: 1
finished w/ slot: 2
finished w/ slot: 3
finished w/ slot: 4
finished w/ slot: 5
finished w/ slot: 6
finished w/ slot: 7
finished w/ slot: 8
finished w/ slot: 9
finished w/ slot: pitcher
finished w/ slot: PH
CPU times: user 493 ms, sys: 3.97 ms, total: 497 ms
Wall time: 499 ms<jupyter_text>Add Pitcher / Batter Matchup History Report<jupyter_code>def add_pb_matchup_priors(pitcher_df, pitcher_df17, start_dates, end_dates):
    df = pd.concat([pitcher_df, pitcher_df17], sort=False)
    
    #initialize empty list to store dfs (concat them together later)
    df_list = []
    
    #iterate over each period
    for i in range(len(start_dates)):
        
        #make the prior and current dfs:
        prior_df = df[df['game_date'] < start_dates[i]]
        current_df = df[(df['game_date'] >= start_dates[i]) & (df['game_date'] <= end_dates[i])]
        
        #get all the pitch_types this pitcher has thrown in the past:
        pitch_types = prior_df['pitch_cat'].unique().tolist()
        
        try:
            pitch_types.remove('PO')
        except:
            pass
        
        #get a list of the batters in the current_df
        current_batters = current_df['batter'].unique().tolist()
        
        batters_dict = {}
        
        current_df_list = []
        
        for batter in current_batters:
            batter_df_list = []
            
            #first use subset from prior df
            batter_subset = prior_df[prior_df['batter'] == batter].copy()
            #if pitcher has never faced this batter before:
            if batter_subset.empty:
                #get the left or right handedness of the batter
                stand = current_df[current_df['batter'] == batter]['stand'].values[0]
                #use overall prior tendencies vs left or right handed hitters
                overall, by_count = get_pitch_tendencies(prior_df[prior_df['stand'] == stand])
            else:
                overall, by_count = get_pitch_tendencies(batter_subset)
            batters_dict[batter] = by_count
            
            #now use subset of current_df where batter=batter
            batter_subset = current_df[current_df['batter'] == batter].copy()
            #iterate over the different count_cat types:
            for count_cat in ['ahead', 'behind', 'neutral']:
                count_subset = batter_subset[batter_subset['count_cat'] == count_cat].copy()
                if count_subset.empty:
                    continue
                else:
                    for pitch in pitch_types:
                        try:
                            count_subset['PB_'+pitch] = batters_dict[batter][count_cat][pitch] * 100
                        except KeyError:
                            count_subset['PB_'+pitch] = 0
                            
                current_df_list.append(count_subset)
        
        current_df = pd.concat(current_df_list, sort=False)
        df_list.append(current_df)
                    
    new_df = pd.concat(df_list, sort=False).sort_values(by=['game_date', 'game_pk', 'pitch_count'])
    return new_df
%%time
pitcher_df = add_pb_matchup_priors(pitcher_df, pitcher_df17, start_dates, end_dates)
pitcher_df17.pitch_type.value_counts()
pitcher_df.pitch_type.value_counts()
pitcher_df17.pitch_cat.value_counts(normalize=True)
pitcher_df.pitch_cat.value_counts(normalize=True)
#test[['pb_FF', 'pb_CH','pb_CU','pb_SL','pb_FT','pb_FC']]<jupyter_output><empty_output><jupyter_text>Choose features to use in model input vector, re-cast datatypes, encode categorical variables (onehot and binary), scale appropriately (Standard vs MinMax vs Robust), and check for any remaining NaNs and choose imputation strategy Train/test Split (IMO use April/2017 - end of June/2019 as Train, and July/August 2019 as Test)<jupyter_code>pitcher_df.head()
model_features = ['game_date', 'batter', 'on_3b', 'on_2b', 'on_1b', 'outs_when_up',
                'inning','fielder_2', 'sz_top', 'sz_bot', 'game_pk','at_bat_number',
                'pitch_number', 'bat_score', 'fld_score', 'stand', 'p_throws',
                'inning_topbot', 'if_fielding_alignment','score_diff', 'on_1b_id',
                'L1_ball_high', 'L1_ball_low', 'L1_ball_left', 'L1_ball_right',
                'bases_loaded', 'fastball_perc_faced', 'fastball_chase_perc',
                'fastball_bip_swung_perc', 'fastball_taken_strike_perc',
                'fastball_est_woba', 'fastball_babip', 'fastball_iso_value',
                'breaking_perc_faced', 'breaking_chase_perc', 'breaking_bip_swung_perc',
                'breaking_taken_strike_perc', 'breaking_est_woba', 'breaking_babip',
                'breaking_iso_value', 'offspeed_perc_faced', 'offspeed_chase_perc',
                'offspeed_bip_swung_perc', 'offspeed_taken_strike_perc',
                'offspeed_est_woba', 'offspeed_babip', 'offspeed_iso_value',
                'pitchout_perc_faced', 'balls','strikes', 'of_fielding_alignment',
                '_count', 'count_cat', 'overall_breaking_perc',
                'count_cat_breaking_perc', 'overall_fastball_perc',
                'count_cat_fastball_perc', 'overall_offspeed_perc',
                'count_cat_offspeed_perc', 'pitch_count', 'L1_pitch_type',
                'L1_pitch_result', 'L1_pitch_zone', 'L5_fastball_perc', 
                'L15_fastball_perc', 'L5_breaking_perc', 'L15_breaking_perc',
                'L5_offspeed_perc', 'L15_offspeed_perc', 'L5_strike_perc',
                'L15_strike_perc', 'batting_order_slot', 'pitcher_AB', 
                'prev_ab_run_scored', 'prev_ab_homerun', 'prev_ab_walk',
                'prev_ab_basehit', 'prev_ab_strikeout', 'PB_fastball', 'PB_breaking',
                'PB_offspeed', 'pitch_cat']

def extract_date_features(df):
    df = df.copy()
    
    #df['day'] = df['game_date'].dt.day
    df['month'] = df['game_date'].dt.month
    df['year'] = df['game_date'].dt.year
    
    df = df.sort_values(by=['game_date', 'game_pk', 'pitch_count'])
    df = df.drop('game_date', axis=1)
    return df
model_df = pitcher_df[model_features].copy()

#rename fielder_2 to catcher_id:
model_df = model_df.rename(columns={'fielder_2':'catcher_id'})

#fillna for on_1b_id:
model_df['on_1b_id'] = model_df['on_1b_id'].fillna(-1)

#fillna for if and of alignment:
model_df[['if_fielding_alignment', 'of_fielding_alignment']] = model_df[['if_fielding_alignment', 'of_fielding_alignment']].fillna('Standard')

#fillna for sz_top and sz_bot
model_df['sz_top'] = model_df['sz_top'].fillna(model_df['sz_top'].mean())
model_df['sz_bot'] = model_df['sz_bot'].fillna(model_df['sz_bot'].mean())

#fillna for catcher_id
catcher = model_df['catcher_id'].value_counts().index[0]
model_df['catcher_id'] = model_df['catcher_id'].fillna(catcher)

#convert datetime into just month and year
model_df = extract_date_features(model_df)

#bool cols replace False and True with 0/1 (and some -1)
bool_cols =  ['L1_ball_high', 'L1_ball_low', 'L1_ball_left', 'L1_ball_right', 
              'bases_loaded', 'pitcher_AB', 'prev_ab_run_scored', 'prev_ab_homerun', 
              'prev_ab_walk', 'prev_ab_basehit', 'prev_ab_strikeout']

model_df[bool_cols] = model_df[bool_cols] * 1
model_df.head(1)
model_df.isna().sum()
import category_encoders as ce
from sklearn.preprocessing import RobustScaler, StandardScaler

#categoricals cols:


#one hot encode categorical columns:
one_hot_cols = ['outs_when_up', 'stand', 'inning_topbot', 'if_fielding_alignment',
                'of_fielding_alignment', 'count_cat', 'L1_pitch_type', 
                'L1_pitch_result', 'L1_ball_high', 'L1_ball_low', 'L1_ball_left', 
                'L1_ball_right','year', 'p_throws']

#binary encode categorical columns:
binary_cols = ['inning', 'batter', 'catcher_id', 'game_pk', 'at_bat_number', 'pitch_number',
               'bat_score', 'fld_score', 'score_diff', 'on_1b_id', 'balls', 'strikes', 
               '_count', 'pitch_count', 'L1_pitch_zone', 'batting_order_slot', 
               'month']
    

#robust scaler cols:

numeric_cols = ['fastball_perc_faced', 'fastball_chase_perc', 'fastball_bip_swung_perc', 
                'fastball_taken_strike_perc', 'fastball_est_woba', 'fastball_babip', 
                'fastball_iso_value', 'breaking_perc_faced', 'breaking_chase_perc', 
                'breaking_bip_swung_perc', 'breaking_taken_strike_perc', 
                'breaking_est_woba', 'breaking_babip', 'breaking_iso_value', 
                'offspeed_perc_faced', 'offspeed_chase_perc', 'offspeed_bip_swung_perc', 
                'offspeed_taken_strike_perc','offspeed_est_woba', 'offspeed_babip', 
                'offspeed_iso_value','overall_breaking_perc', 'count_cat_breaking_perc', 
                'overall_fastball_perc', 'count_cat_fastball_perc', 'overall_offspeed_perc', 
                'count_cat_offspeed_perc', 'pitchout_perc_faced', 'L5_fastball_perc', 
                'L15_fastball_perc', 'L5_breaking_perc', 'L15_breaking_perc',
                'L5_offspeed_perc', 'L15_offspeed_perc', 'L5_strike_perc', 
                'L15_strike_perc', 'PB_fastball', 'PB_breaking', 'PB_offspeed', 
                'sz_bot', 'sz_top']

def final_model_preprocess(model_df):
    df = model_df.copy()
    
    #one hot encode certain columns
    one_hot_ce = ce.OneHotEncoder(cols=one_hot_cols, drop_invariant=False, 
                                  return_df=True, verbose=10, use_cat_names=True)
    
    one_hot_df = df[one_hot_cols].copy() 
    one_hot_df = one_hot_ce.fit_transform(one_hot_df)
    
    #binary encode columns w/ lots of categories
    binary_ce = ce.BinaryEncoder(cols=binary_cols, return_df=True)
    
    binary_df = df[binary_cols].copy()
    binary_df = binary_ce.fit_transform(binary_df)
    
    #grab the remaining features
    other_cols = set(df.columns) - set(one_hot_cols + binary_cols)
    other_df = df[other_cols].copy()
    
    #merge everything back together
    df = pd.concat([one_hot_df, binary_df, other_df], axis=1)
    return df
%%time
model_df = final_model_preprocess(model_df)
model_df.shape
model_df.head()
#Todo:

# - pickle the model_df and export

# - scale the data (probably w/ robust scaler)

# - split into train and test

# - train models and evaluate predictions
model_df.to_pickle(path=("verlander_df.pkl"),compression='zip')
!ls<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11329800656068635, 'fraction_numerical': 0.018515013878374967, 'mean_word_length': 2.878761928064595, 'pattern_counts': {'":': 0, '<': 59, '<?xml version=': 0, '>': 54, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 3, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '7', 'dir': 'jupyter-structured-clean-dedup', 'id': '177311', 'license': 'MIT', 'n_tokens_mistral': 11214, 'n_tokens_neox': 10356, 'n_words': 2070, 'path': 'notebooks/Pitcher_model_pre_processing(9-10_update_ready_for_modelling).ipynb', 'repo_name': 'labs15-baseball-pitch-predictor/RC-v1.1--Data-Cleaning-and-Feature-Engineering', 'size': '31824'}