<jupyter_start><jupyter_text>Main Pre-processing functions:<jupyter_code>def downcast_dtypes(df):
    df = df.copy()
    int_cols = df.select_dtypes('int').columns.tolist()
    float_cols = df.select_dtypes('float').columns.tolist()
    obj_cols = df.select_dtypes('object').columns.tolist()
    cat_cols = []
    for col in obj_cols:
        if col == 'pitch_type':
            continue
        if len(df[col].unique()) < len(df)/2:
            cat_cols.append(col)
      
    ints = df[int_cols].apply(pd.to_numeric,downcast='unsigned')
    floats = df[float_cols].apply(pd.to_numeric,downcast='float')
    cats = df[cat_cols].astype('category')
  
    df = df.drop(columns=int_cols + float_cols + cat_cols)
    for d in [ints, floats, cats]:
        df = pd.concat([df, d], axis=1)
    return df

def make_game_features(df):
    df = df.copy()
    
    #convert balls and strikes columns to strings (first to int)
    df['balls'] = df['balls'].astype('int').astype('str')
    df['strikes'] = df['strikes'].astype('int').astype('str')

    #create count feature:
    df['_count'] = df['balls'] + df['strikes']

    #create count category feature:
    count_map = {'00':'neutral', '21':'neutral', '32':'neutral', '10':'behind', '20':'behind',
             '30':'behind', '31':'behind', '01':'ahead', '02':'ahead', '11':'ahead',
             '12':'ahead', '22':'ahead'}
    df['count_cat'] = df['_count'].replace(count_map)
    
    #cant decide whether to drop the balls and strikes feature, keeping for now- check feature importances later
    #df = df.drop(columns=['balls', 'strikes'])
    
    #make score_differential feature
    df['score_diff'] = df['fld_score'] - df['bat_score']
    
    #make baserunners features 1/0 instead of baserunner id:
    df['on_1b_id'] = df['on_1b'] #save the runners id in case we later add feature for speedy runners
    df['on_1b'] = df['on_1b'] * 0 + 1
    df['on_1b'] = df['on_1b'].fillna(0)
    df['on_2b'] = df['on_2b'] * 0 + 1
    df['on_2b'] = df['on_2b'].fillna(0)
    df['on_3b'] = df['on_3b'] * 0 + 1
    df['on_3b'] = df['on_3b'].fillna(0)
    
    #make bases_loaded feature:
    df['bases_loaded'] = df['on_1b'] + df['on_2b'] + df['on_3b']
    df['bases_loaded'] = df['bases_loaded'].apply(lambda x: 1 if x == 3 else 0)

    return df

def gen_pitcher_percentages(df):
    df = df.copy()
    #get all the unique pitcher names in the df
    pitcher_list = df.pitcher.unique().tolist()
    #initialize empty dictionary to store each pitcher and their pitches and percentages for each pitch
    pitcher_dict = {}
    #iterate over each pitcher:
    for pitcher in pitcher_list:
        #assign the normalized value_counts to a variable
        pitch_percentages = df[df.pitcher == pitcher].pitch_type.value_counts(normalize=True)
        #convert that Series object to a dict and assign it as the value to the pitcher dictionary 
        #(pitcher name as key)
        pitcher_dict[pitcher] = pitch_percentages.to_dict()
    return pitcher_dict

def fill_pitch_type_nans(df, pitcher_dict):
    df = df.copy()
    #grab the rows where pitch_type is null:
    nulls = df[df.pitch_type.isna()]
    #For pitchers w/ only NaNs for pitch type, just delete them
    delete_idx_list = []
    
    #iterate over each null row
    for index, row in nulls.iterrows():
        #use the % for that pitcher for each pitch type he throws to generate a random pitch type with 
        #that % as weight
        try:
            pitch = random.choices(population=list(pitcher_dict[row.pitcher].keys()), 
                               weights=list(pitcher_dict[row.pitcher].values()), 
                               k=1)[0]
            #fill the NaN value with the randomly generated pitch
            df.at[index, 'pitch_type'] = pitch
        except KeyError:
            try:
                pvc = df[df['pitcher']==row.pitcher]['pitch_type'].value_counts(normalize=True)
                pitch_types = pvc.index.tolist()
                percentages = pvc.values.tolist()
                pitch = random.choices(population=pitch_types, weights=percentages, k=1)[0]
            except IndexError:
                print('IndexError: deleting ' + str(row.pitcher))
                delete_idx_list.append(index)
                
        except IndexError:
            print('IndexError: ' + str(row.pitcher))
    
    #drop the pitchers w/ all NaNs before returning       
    df = df.drop(delete_idx_list)
    return df

def make_pitch_type_cat(df):
    df = df.copy()
    
    #create map for pitch type into categories:
    pitch_type_map = {'FA':'fastball', 'FF':'fastball', 'FT':'fastball', 'FC':'fastball',
                      'FS':'fastball', 'SI':'fastball', 'SF':'fastball', 'SL':'breaking',
                      'CB':'breaking', 'CU':'breaking', 'SC':'breaking', 'KC':'breaking',
                      'CH':'offspeed', 'KN':'offspeed', 'EP':'offspeed', 'FO':'breaking', 
                      'PO':'pitchout'}

    #create pitch cateogory feature
    df['pitch_cat'] = df['pitch_type']
    df['pitch_cat'] = df['pitch_cat'].replace(pitch_type_map)
    return df

def make_strikezone_swung_and_chase_features(df):
    df = df.copy()
    
    #create swung column
    def swung(x):
        swung = ['foul','hit_into_play','swinging_strike','hit_into_play_no_out',
                 'hit_into_play_score','foul_tip','swinging_strike_blocked',
                 'foul_bunt','missed_bunt']
        return 1 if x in swung else 0
    df['batter_swung'] = df['description'].apply(swung)
    
    #initialize in_strikezone and chased features:
    df['in_strikezone'] = 1
    df['chased'] = 0
    
    df['ball_high'] = df['plate_z'] > df['sz_top']
    df['ball_low'] = df['plate_z'] < df['sz_bot']
    df['ball_left'] = df['plate_x'].apply(lambda x: x < -0.73)
    df['ball_right'] = df['plate_x'].apply(lambda x: x > 0.73)
    df['in_strikezone'] = df['ball_high'] + df['ball_low'] + df['ball_left'] + df['ball_right']
    df['in_strikezone'] = df['in_strikezone'].apply(lambda x: 0 if x > 0 else 1)
    
    nulls_subset = df[df['in_strikezone'].isna()]
    for index, row in nulls_subset.iterrows():
        if row.type == 'B':
            df.at[index, 'in_strike_zone'] = False
    
    df['chased'] = df['batter_swung'] - df['in_strikezone']
    df['chased'] = df['chased'].apply(lambda x: 1 if x == 1 else 0)
    return df

#change this to return the batters_df (and pass in prior)
def make_batters_df(prior_df):
    df = prior_df.copy()
    #make list of the unique batter ids
    batters = list(df['batter'].unique())
    #initialize empty dictionary to store the batter stats
    batters_dict = {}
    #set a break flag to False for error-checking
    brk = False
    #iterate thru each unique batter
    for batter in batters:
        if brk:
            break
        #make subset of the df for that batter and assign to variable batter_df    
        batter_df = df[df['batter'] == batter]
        
        #assign all pitch categories to list:
        all_pitch_cats = ['fastball', 'breaking', 'offspeed', 'pitchout']
        
        #assign the pitch categories to a list
        try:
            pitch_cats = batter_df['pitch_cat'].unique().tolist()
        except KeyError:
            print(batter)
            brk = True
        #get the normalized value counts of pitches by category that batter has faced
        vc = batter_df.pitch_cat.value_counts(normalize=True)
        #initialize empty dict for each batter
        batter_dict = {}
        
        #if there are any pitch categories the batter has not faced, 
        unfaced_cats = list(set(all_pitch_cats) - set(pitch_cats))
        
        for cat in pitch_cats:
            if brk:
                break
        
            #assign the % of pitches faced by the batter for that category to his batter dict
            try:
                batter_dict[cat + '_perc_faced'] = vc[cat] * 100
            except TypeError:
                print(batter)
                return 1
                
            #continue out of the loop for pitchout category since ball in play stats are NaN
            if cat == 'pitchout':
                continue
        
            #grab subset of batter df for the pitch category
            cat_df = batter_df[batter_df['pitch_cat'] == cat]
            
            #if he has faced less than 100 pitches of that type, add it to unfaced_category and fill w NaN
            if len(cat_df) < 100:
                unfaced_cats.append(cat)
                continue
        
            #calculate batters chase % for pitch type category on balls outside the strikezone
            out_of_strikezone = len(cat_df) - cat_df['in_strikezone'].sum() #num of times ball was out of zone
            chased_count = cat_df['chased'].sum() #num of times batter chased
            try:
                chase_perc = (chased_count / out_of_strikezone) * 100
            except ZeroDivisionError:
                chase_perc = np.nan
            #assign the chase perc to the batter dict
            batter_dict[cat + '_chase_perc'] = chase_perc
        
            #calc ball in play % for each swing for each pitch cat:
            ball_in_play_count = len(cat_df[cat_df['type'] == 'X']) #type X means ball hit into play
            swung_count = cat_df['batter_swung'].sum() #counts all the 1s in the swung column
            #assign the ball in play % per swing to the batter dict
            batter_dict[cat + '_bip_swung_perc'] = (ball_in_play_count / swung_count) * 100
        
            #calculate taken strike %
            taken_strike_count = len(cat_df[(cat_df['in_strikezone'] == 1) & (cat_df['batter_swung'] == 0)])
            pitches_in_zone_count = cat_df['in_strikezone'].sum() #counts the 1s in the in zone col
            #assign to batter_dict
            batter_dict[cat + '_taken_strike_perc'] = (taken_strike_count / pitches_in_zone_count) * 100
        
            #for each pitch type category, get the batters stats on balls hit in play
            stats = ['estimated_woba_using_speedangle', 'babip_value', 'iso_value']
            for stat in stats:
                #drop Nans from the stat column and assign to new subset, for each stat
                stat_cat_df = cat_df.dropna(subset=[stat])
                if stat == 'estimated_woba_using_speedangle':
                    #get the mean avg_est_woba
                    avg_est_woba = stat_cat_df['estimated_woba_using_speedangle'].mean()
                    #assign that value to the batters dictionary
                    batter_dict[cat + '_est_woba'] = avg_est_woba
                    if avg_est_woba == np.nan:
                        print(batter)
                        brk = True
                        break
                elif stat == 'babip_value':
                    avg_babip = stat_cat_df['babip_value'].mean()
                    batter_dict[cat + '_babip'] = avg_babip
                else:
                    avg_iso_value = stat_cat_df['iso_value'].mean()
                    batter_dict[cat + '_iso_value'] = avg_iso_value
                    
        #for unfaced or small sample pitch_types: assign NaNs to his dictionary for that category
        for cat in unfaced_cats:
            if cat == 'pitchout':
                batter_dict[cat + '_perc_faced'] = 0
            else:
                batter_dict[cat + '_perc_faced'] = np.nan
                batter_dict[cat + '_chase_perc'] = np.nan
                batter_dict[cat + '_bip_swung_perc'] = np.nan
                batter_dict[cat + '_taken_strike_perc'] = np.nan
                batter_dict[cat + '_est_woba'] = np.nan
                batter_dict[cat + '_babip'] = np.nan
                batter_dict[cat + '_iso_value'] = np.nan
        
        #assign the batter dictionary to the main dictionary of all batters
        batters_dict[batter] = batter_dict
    if not brk:
        print('iteration completed successfully')
        
    #make df from the batters dict
    batters_df = pd.DataFrame.from_dict(batters_dict, orient='index')
    batters_df = batters_df.reset_index().rename(columns={'index':'batter'})
    return batters_df<jupyter_output><empty_output><jupyter_text>Read in the data and downcast the dtypes<jupyter_code>drop_cols = ['pitcher.1','fielder_2.1', 'fielder_3', 'fielder_4', 'fielder_5', 'fielder_6', 'fielder_7', 'fielder_8','fielder_9']

# df_16 = pd.read_pickle('pitches_2016.pkl', compression='zip').drop(columns=drop_cols)
# df_16 = downcast_dtypes(df_16)
# df_16.head()
df_17 = pd.read_pickle('pitches_2017.pkl', compression='zip').drop(columns=drop_cols)
df_17 = downcast_dtypes(df_17)
df_17.head()
df_18 = pd.read_pickle('pitches_2018.pkl', compression='zip').drop(columns=drop_cols)

#drop a particular pitcher w/ only 16 pitches (all NaNs)
drop_idx = df_18[df_18['pitcher'] == 664068.0].index
df_18 = df_18.drop(drop_idx)

df_18 = downcast_dtypes(df_18)
df_18.head()
df_19 = pd.read_pickle('pitches_2019.pkl', compression='zip').drop(columns=drop_cols)
df_19.head()<jupyter_output><empty_output><jupyter_text>Combine 2017 and 2018 and 2019 into one df and clean up some faulty data<jupyter_code>combined = pd.concat([df_17, df_18, df_19], sort=False)
combined = combined.sort_values(by = ['game_date', 'game_pk', 'at_bat_number', 'pitch_number']).reset_index(drop=True)
%%time
#iterate over every game and find the ones w/ all Nans for pitch_type and delete them from combined df
games = combined.game_pk.unique().tolist()
for game in games:
    game_df = combined[combined.game_pk == game]
    #if the game is all Nans for pitch_type
    if len(game_df.pitch_type.value_counts()) == 0:
        #delete it from combined
        combined = combined.drop(game_df.index)
        
#reset index again:
combined = combined.reset_index(drop=True)
combined.head()
#If using 2016 data:

#fix some bad data for pitch_type

# pitch_types = ['FF','SL', 'FT', 'CH', 'SI', 'CU', 'FC', 'KC', 'FS', 'KN', 'IN', 'FO', 'PO','EP','SC','UN', 'FA']
# bad_data = combined[~(combined.pitch_type.isin(pitch_types))].pitch_type.value_counts().index.tolist()
# combined['pitch_type'] = combined.pitch_type.replace(to_replace=bad_data, value=np.nan)
combined.info()
#clear some memory:
import gc

#del df_16
del df_17
del df_18
del df_19

gc.collect()<jupyter_output><empty_output><jupyter_text>2-step pre-processing functions:Step 1 features for all rows:Step 2 features that iterate over each month of 2017 and 2018 so that aggregates use prior data only (preventing leakage of future data into the training set)<jupyter_code>def pre_process_step1(combined):
    df = combined.copy()
    #convert the pitch type for UN (unknown) to np.nan
    df['pitch_type'] = df['pitch_type'].replace({'UN':np.nan})
    #fix some faulty data that has number of balls listed as 4:
    df['balls'] = df['balls'].replace({4.0: 3.0})
    
    #count, count_cat, score_diff, on_base 1/0, bases_loaded
    df = make_game_features(df)
    
    #batter_swung, in_strikezone, chased
    df = make_strikezone_swung_and_chase_features(df)
    
    #get aggregate pitcher %s dict from prior data:
    pitcher_dict = gen_pitcher_percentages(df)
    
    #fil the NaNs for pitch_type using randomized guess from pitcher tendencies
    df = fill_pitch_type_nans(df, pitcher_dict)
        
    #pitch_type category feature
    df = make_pitch_type_cat(df)
    
    return df

#pass in list of periods to update the data (and fill NaNs) using prior aggregates:
def pre_process_step2(pre_processed_step1, start_dates, end_dates):
    df = pre_processed_step1.copy()
    
    #initialize empty list to store dfs (concat them together later)
    df_list = []
    
    #iterate over each period
    for i in range(len(start_dates)):
        #make the prior and current dfs:
        prior_df = df[df['game_date'] < start_dates[i]]
        current_df = df[(df['game_date'] >= start_dates[i]) & (df['game_date'] <= end_dates[i])]
        
        #add the batter scouting report
        batters_df = make_batters_df(prior_df)
        current_df = pd.merge(current_df, batters_df, how='left', on='batter')
        
        #append the df to the list
        df_list.append(current_df)
    
    step2_df = pd.concat(df_list, sort=False)
    return step2_df
#2017 season: '2017-04-02', '2017-10-02'
#2018 season: '2018-03-29' - '2018-10-01'
#2019 season: '2019-03-28' - '2018-08-31'

#if we decide to use 2.5 seasons of training instead of 1.5, here the dates for 2017:
#2017_start_dates = ['2017-04-02', '2017-05-01', '2017-06-01', '2017-07-01', '2017-08-01', '2017-09-01']
#2017_end_dates = ['2017-04-30', '2017-05-31', '2017-06-30', '2017-07-31', '2017-08-31', '2017-10-02']

start_dates = ['2018-03-29', '2018-05-01', '2018-06-01', '2018-07-01', '2018-08-01', '2018-09-01', '2019-03-28', '2019-05-01', '2019-06-01', '2019-07-01', '2019-08-01']
end_dates =  ['2018-04-30', '2018-05-31', '2018-06-30', '2018-07-31', '2018-08-31', '2018-10-01', '2019-04-30', '2019-05-31', '2019-06-30', '2019-07-31', '2019-08-31']
%%time
step1_df = pre_process_step1(combined)
#clear some memory
#del combined
#gc.collect()
step1_df.head()
len(step1_df)
%%time
step2_df = pre_process_step2(step1_df, start_dates, end_dates)
step2_df.head()
len(step2_df)
step2_df.isna().sum()
#sort and reset index
step1_df = step1_df.sort_values(by = ['game_date', 'game_pk', 'at_bat_number', 'pitch_number']).reset_index(drop=True)
step2_df = step2_df.sort_values(by = ['game_date', 'game_pk', 'at_bat_number', 'pitch_number']).reset_index(drop=True)

index_middle = len(step2_df) // 2
step2_df1 = step2_df.loc[:index_middle]
step2_df2 = step2_df.loc[index_middle:]

step2_df1.tail()
step2_df2.head()
#export pre_processed_df to pkl
#step1_df.to_pickle(path=("step1_pre_processed_df.pkl"),compression='zip')
step2_df1.to_pickle(path=("pre_pitcher_pre_processed_df_part1.pkl"),compression='zip')
step2_df2.to_pickle(path=("pre_pitcher_pre_processed_df_part2.pkl"),compression='zip')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1263017207774127, 'fraction_numerical': 0.03391435094949045, 'mean_word_length': 2.8586162440911043, 'pattern_counts': {'":': 0, '<': 23, '<?xml version=': 0, '>': 21, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '103041', 'license': 'MIT', 'n_tokens_mistral': 6494, 'n_tokens_neox': 5887, 'n_words': 1458, 'path': 'notebooks/Pre_pitcher_preprocessing_script.ipynb', 'repo_name': 'labs15-baseball-pitch-predictor/RC-v1.1--Data-Cleaning-and-Feature-Engineering', 'size': '17957'}