<jupyter_start><jupyter_text>Assumptions- We will calculate this for top 50 batters every day- We have the starters for today Stats Sets We Need- batter recent performance    - batter_saber_team.parquet- batter performance against pitcher    - innings.parquet- batter performance against opponent bullpen    - innings.parquet- starter    - innings.parquet    - preview_pull- starter recent performance    - pitcher_saber_team.parquet- starter avg innings per game    - pitcher_saber_team.parquet    - innings.parquet    - pitching.parquet- target    - innings.parquet  Batter Recent Performance<jupyter_code>path = CONFIG.get('paths').get('batter_saber')
dd_list = [p for p in os.listdir(path)]
df_bat_perf = pd.concat(
    objs=[pd.read_parquet(path+"{}/".format(dd)+'batter_saber_team.parquet') for dd in dd_list 
          if "batter_saber_team.parquet" in os.listdir(path+"{}/".format(dd))],
    axis=0
)
df_bat_perf['flag'] = pd.isnull(df_bat_perf).any(axis=1)
df_bat_perf = df_bat_perf.loc[df_bat_perf['flag'] == False, :]
df_bat_perf.drop(labels=['flag'], axis=1, inplace=True)
df_bat_perf = df_bat_perf.loc[:, [
    'gameId', 'gameDate', 'team', 'batterId', 
    'batterWalkPercentage_trail3',
    'batterWalkPercentage_trail6', 'batterKPercentage_trail3',
    'batterKPercentage_trail6', 'batterISO_trail3', 'batterISO_trail6',
    'batterBABIP_trail3', 'batterBABIP_trail6', 'woba_trail3',
    'woba_trail6', 'ab_trail3', 'ab_trail6'
]]
df_bat_perf.head(10)<jupyter_output><empty_output><jupyter_text>Batter Against Pitcher<jupyter_code>path = CONFIG.get('paths').get('normalized')
years = [x.split("_")[1][:4] for x in os.listdir(path)]
years = list(set(years))
print(years)
df_inning_at = []
for yr in years:
    dd_list = [x for x in os.listdir(path) if yr in x]
    df_bm_all = pd.concat(
        objs=[pd.read_parquet(path+"{}/".format(dd)+'innings.parquet') for dd in dd_list 
              if "innings.parquet" in os.listdir(path+"{}/".format(dd))],
        axis=0
    )
    cols = ['atbat_batter', 'atbat_event', 'atbat_event_num', 
            'atbat_pitcher', 'game_id', 'gameId', 'gameDate', 
            'home_starting_pitcher', 'away_starting_pitcher',
            'inning_half']
    df_bm_all = df_bm_all.loc[:, cols]
    df_bm_all['year'] = yr
    df_bm_all = df_bm_all.drop_duplicates(inplace=False)
    df_bm_all = df_bm_all.loc[df_bm_all['gameId'].notnull(), :]
    df_inning_at.append(df_bm_all)
df_bat_mtch = pd.concat(df_inning_at, axis=0)
df_bat_mtch = df_bat_mtch.drop_duplicates(inplace=False)
df_bat_mtch.loc[:, 'hr_flag'] = (
    df_bat_mtch.atbat_event.str.replace(" ", "").\
    str.lower().str.contains('homerun').astype(float)
)

print("atbat_side")
# Add flag for if atbat batterId is home or away for game
df_bat_mtch.loc[:, 'atbat_side'] = np.NaN
df_bat_mtch.loc[df_bat_mtch['inning_half'] == 'top', 'atbat_side'] = 'away'
df_bat_mtch.loc[df_bat_mtch['inning_half'] == 'bottom', 'atbat_side'] = 'home'
#assert all(df_bat_mtch['atbat_side'].notnull())

df_bat_mtch.reset_index(drop=True, inplace=True)
print("atbat_opp_starter")
# Add atbat_opp_starter depending on what team batter is on
df_bat_mtch.loc[:, 'atbat_opp_starter'] = np.NaN
df_bat_mtch.loc[df_bat_mtch['inning_half'] == 'top', 'atbat_opp_starter'] = df_bat_mtch['home_starting_pitcher']
df_bat_mtch.loc[df_bat_mtch['inning_half'] == 'bottom', 'atbat_opp_starter'] = df_bat_mtch['away_starting_pitcher']
df_bat_mtch.shape
print('done all')
df_bat_mtch[['gameId', 'atbat_batter', 'atbat_event_num', 'hr_flag']].head(10)<jupyter_output><empty_output><jupyter_text>Calculations Calc: Batter Recent Performance<jupyter_code># Recent Performance for this year<jupyter_output><empty_output><jupyter_text>Calc: Batter Against Pitcher This Year<jupyter_code># Atbats
df_bat_rec = df_bat_mtch.loc[df_bat_mtch['year'] == dd[:4], :]



df_bat_rec.loc[:, 'year'] = df_bat_rec['gameDate'].dt.year.astype(str)
df_bat_rec_agg = df_bat_rec.groupby(
    by=['year', 'gameId', 'atbat_batter', 'atbat_opp_starter'],
    as_index=False
).agg({'hr_flag': 'max',
       'atbat_event_num': 'count'})
df_bat_rec_agg.rename(columns={'atbat_event_num': 'atbat_count',
                               'hr_flag': 'hr_count'}, 
                     inplace=True)

# Bring to Year Level
# Sort
df_bat_rec_agg.sort_values(
    by=['year', 'gameId', 'atbat_batter', 'atbat_opp_starter'],
    ascending=True, 
    inplace=True
)
# Cumulative Sums for each year
df_bat_rec_agg['hr_count_roll'] = (
    df_bat_rec_agg.groupby(['year', 'atbat_batter', 'atbat_opp_starter'])
    ['hr_count'].cumsum().reset_index(drop=True)
)
df_bat_rec_agg['atbat_count_roll'] = (
    df_bat_rec_agg.groupby(['year', 'atbat_batter', 'atbat_opp_starter'])
    ['atbat_count'].cumsum().reset_index(drop=True)
)
df_bat_rec_agg.loc[:, 'hr_prop'] = (
    df_bat_rec_agg['hr_count_roll'] / 
    df_bat_rec_agg['atbat_count_roll']
)
df_bat_rec_agg.tail(5)<jupyter_output>/anaconda2/envs/litesaber_jupyter/lib/python3.6/site-packages/pandas/core/indexing.py:537: SettingWithCopyWarning: 
A value is trying to be set on a copy of a slice from a DataFrame.
Try using .loc[row_indexer,col_indexer] = value instead

See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy
  self.obj[item] = s<jupyter_text>Calc: Batter Against Pitcher All Time<jupyter_code># Atbats
df_bat_rec_agg_alltime = df_bat_mtch.loc[df_bat_mtch['year'].astype(int) < int(dd[:4]), :]


df_bat_rec_agg_alltime = df_bat_rec_agg_alltime.groupby(
    by=['gameId', 'atbat_batter', 'atbat_opp_starter'],
    as_index=False
).agg({'hr_flag': 'max',
       'atbat_event_num': 'count'})
df_bat_rec_agg_alltime.rename(columns={'atbat_event_num': 'atbat_count',
                               'hr_flag': 'hr_count'}, 
                     inplace=True)

# Bring to Year Level
# Sort
df_bat_rec_agg_alltime.sort_values(
    by=['gameId', 'atbat_batter', 'atbat_opp_starter'],
    ascending=True, 
    inplace=True
)
# Cumulative Sums for each year
df_bat_rec_agg_alltime['hr_count_roll'] = (
    df_bat_rec_agg_alltime.groupby(['atbat_batter', 'atbat_opp_starter'])
    ['hr_count'].cumsum().reset_index(drop=True)
)
df_bat_rec_agg_alltime['atbat_count_roll'] = (
    df_bat_rec_agg_alltime.groupby(['atbat_batter', 'atbat_opp_starter'])
    ['atbat_count'].cumsum().reset_index(drop=True)
)
df_bat_rec_agg_alltime.loc[:, 'hr_prop'] = (
    df_bat_rec_agg_alltime['hr_count_roll'] / 
    df_bat_rec_agg_alltime['atbat_count_roll']
)
df_bat_rec_agg_alltime.tail(5)<jupyter_output><empty_output><jupyter_text>Calc: Batter Against Pitcher All-Time  Batter Performance Against Opponent Bullpen This Year  Shift Target back (so next game HR flag becomes current target)<jupyter_code># Aggregate for flag of any HR per game
target = df_bat_mtch.groupby(
    by=['atbat_batter', 'gameId', 'gameDate'],
    as_index=False
).agg({'hr_flag': 'max'})

# Shift back gameId to make the HR count reflect the game that followed
target.loc[:, 'gameId_back'] = np.NaN
target.loc[(target['atbat_batter'] == target['atbat_batter'].shift(1)), 'gameId_back'] = target['gameId'].shift(1)
target.rename(columns={'hr_flag': 'hr_flag_next_game'})
target.drop(labels=['gameId', 'gameDate'], axis=1, inplace=True)<jupyter_output><empty_output><jupyter_text>Combine Features<jupyter_code># Add starter to batter recent agg
df_ft = df_bat_mtch.loc[:, ['gameId', 'gameDate', 'atbat_batter', 'atbat_opp_starter']].drop_duplicates(inplace=False)
df_ft.sort_values(by=['gameId', 'atbat_batter', 'atbat_opp_starter'], ascending=True, inplace=True)
df_ft.drop_duplicates(subset=['gameId', 'atbat_batter'], inplace=True)

df_ft.sort_values(by=['gameDate'], ascending=True, inplace=True)
df_ft = pd.merge(
    df_ft,
    df_bat_perf,
    how='inner',
    left_on=['gameId', 'gameDate', 'atbat_batter'],
    right_on=['gameId', 'gameDate', 'batterId'],
    validate='1:1'
)
df_ft.drop(labels=['batterId'], axis=1, inplace=True)
#df_bat_rec_agg.rename(
#    columns={x: x+"_mtchup_starter" for x in df_bat_rec_agg.columns 
#             if x not in ['gameId', 'atbat_batter', 'atbat_opp_starter', 'year']},
#    inplace=True
#)
df_ft = pd.merge(
    df_ft,
    df_bat_rec_agg,
    how='inner',
    left_on=['gameId', 'atbat_batter', 'atbat_opp_starter'],
    right_on=['gameId', 'atbat_batter', 'atbat_opp_starter'],
    validate='1:1'
)
print(df_ft.shape)

# Merge on Target
df_ft = pd.merge(
    df_ft,
    target,
    how='inner',
    left_on=['gameId', 'atbat_batter'],
    right_on=['gameId_back', 'atbat_batter'],
    validate='1:1'
)
df_ft.drop(labels=['gameId_back'], axis=1, inplace=True)
print(df_ft.shape)
df_ft.tail()
if not os.path.exists("/Volumes/Samsung_T5/mlb/gdApi/99_hr_prop_ft/"):
    os.makedirs("/Volumes/Samsung_T5/mlb/gdApi/99_hr_prop_ft/")
df_ft.to_parquet('/Volumes/Samsung_T5/mlb/gdApi/99_hr_prop_ft/featurespace_all.parquet')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1516103692065986, 'fraction_numerical': 0.0061721467848726294, 'mean_word_length': 5.538517975055026, 'pattern_counts': {'":': 0, '<': 29, '<?xml version=': 0, '>': 28, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '7', 'dir': 'jupyter-structured-clean-dedup', 'id': '242396', 'license': 'MIT', 'n_tokens_mistral': 3618, 'n_tokens_neox': 3520, 'n_words': 535, 'path': 'data/scrap/home_run_prop.ipynb', 'repo_name': 'PeterA182/liteSaber', 'size': '8911'}