<jupyter_start><jupyter_text>Hot/cold streak definition* Defined a hot / cold wOBA threshold * Rolling wOBA over five-game periods (so first or last five of the year cant be considered)* Must have at least 20 PA in the five game stretch to be considered* Thresholds defined as groups where 5 percent of league is hot, 5 is cold for a given 5 game stretch* Roughly this means .525 woba for hot, .195 for cold TODO: make sure streaks don't count if a player missed a game in the middle.<jupyter_code># define woba numerator and denominator to get N-game woba
# weights by year https://www.fangraphs.com/guts.aspx?type=cn
df = df %>% mutate(
    woba_numerator = 0.9*B_1B + 1.24*B_2B + 1.56*B_3B + 1.95*B_HR + 0.72*(B_BB - B_IBB) + 0.75*B_HP +  0.92*B_XI, # the book's weights
    woba_denominator = B_PA,
    game_woba = woba_numerator / woba_denominator)
# SQL query to create players' expected woba
# expected woba: a player's woba from the current, previous, and following season, omitting the player's past 5 games (inclusive of the current game)
player_woba = sqldf('
      select 
        df1."person.key",
        df1.season,
        df1.game_num,
        SUM(coalesce(df2.woba_numerator, 0)) season_woba_num,
        SUM(coalesce(df2.woba_denominator, 0)) season_woba_denom,
        sum(coalesce(df2.woba_numerator, 0))/sum(coalesce(df2.woba_denominator, 0)) expected_woba,
        sum(df2.B_PA) season_pa
      from
        df AS df1
      left join
        (select "person.key", season, game_num, SUM(coalesce(woba_numerator, 0)) woba_numerator,
    SUM(coalesce(woba_denominator, 0)) woba_denominator, sum(B_PA) B_PA from df group by 1,2,3) AS df2
      ON
        df1."person.key" = df2."person.key"
      AND
        df1.season <= df2.season + 1
      AND
        df1.season >= df2.season - 1
      AND
        (
            (df1.season*1000 + df1.game_num) <= (df2.season*1000 + df2.game_num -7)
        OR
            (df1.season*1000 + df1.game_num) > (df2.season*1000 + df2.game_num)
        )
      GROUP BY df1."person.key", df1.season, df1.game_num
')
player_woba %>% filter(season_pa>50) %>% arrange(-expected_woba) %>% head(15)
df = df %>% left_join(player_woba,by=c('season', 'person.key', 'game_num'))

# remove these seasons. no future/past season available for calculating their expected woba values
df = df %>% filter(season > 1999, season < 2019)

dim(df)
# remove players we can't calculate an expected woba for 
df = df %>% inner_join(player_years_to_keep, by=c("person.key", "season"), suffixes=c("", "_ignore"))
dim(df)
# take rolling sums of woba components to get a 5g woba
library(zoo)
num_games_streak = 5
num_games_post_streak = 5
min_pa_in_streak = 4 * num_games_streak #4
min_pa_post_streak = 3 * num_games_post_streak #0
# different offset used for post streak folling counts for 3 vs. 5 game configuration. paste this in if changing streak lenghts.
seven_game_post = list(c(1, 2, 3,4,5,6,7))
five_game_post = list(c(1, 2, 3, 4, 5))

# this is ugly indentation but makes it ahrder to forget to paste something when updating it
df = df %>%
         arrange(season, person.key, game.date, game.key) %>%
         group_by(season, person.key) %>%
         mutate(woba_num_rolling =   roll_sum(woba_numerator, num_games_streak, align = "right", fill = NA),
                pa_rolling =         roll_sum(B_PA, num_games_streak, align = "right", fill = NA),
                woba_denom_rolling = roll_sum(woba_denominator, num_games_streak, align = "right", fill = NA),
                woba_rolling = woba_num_rolling/woba_denom_rolling,
                woba_num_rolling_future =   rollapply(woba_numerator, FUN=sum, five_game_post, align = "left", fill = NA),
                pa_rolling_future =         rollapply(B_PA, FUN=sum, five_game_post, align = "left", fill = NA),
                woba_denom_rolling_future = rollapply(woba_denominator, FUN=sum,  five_game_post, align = "left", fill = NA),
                woba_rolling_future = woba_num_rolling_future/woba_denom_rolling_future,
                
                woba_num_rolling_future_nextgame =   rollapply(woba_numerator, FUN=sum, list(c(1)), align = "left", fill = NA),
                woba_denom_rolling_future_nextgame  = rollapply(woba_denominator, FUN=sum,  list(c(1)), align = "left", fill = NA),
                woba_rolling_future_nextgame  = woba_num_rolling_future_nextgame /woba_denom_rolling_future_nextgame 
        )<jupyter_output>Attaching package: ‘zoo’


The following objects are masked from ‘package:base’:

    as.Date, as.Date.numeric<jupyter_text>Get Thresholds for Hot/Cold Streaks (top/bottom 5% of league in x-game rolling wOBA)<jupyter_code># get seasons' percentiles for woba thresholds
df %>% 
    # only consider periods with at least 20 pa in them 
    filter(pa_rolling >= min_pa_in_streak, pa_rolling_future >= min_pa_post_streak) %>% 
    group_by(season, person.key, game.key) %>% summarise(woba = sum(woba_numerator) / sum(woba_denominator),
                                                  #woba_rolling_avg = mean(woba_rolling, na.rm=TRUE), 
                                                  woba_rolling_avg = sum(woba_num_rolling, na.rm=TRUE)/sum(woba_denom_rolling, na.rm=TRUE),
                                                  pa = sum(B_PA)) %>%
    ungroup() %>%
    summarise(thresholds = quantile(woba_rolling_avg, probs = c(0.05, 0.95))) 
# create is hot/cold flags
hot_threshold = .562
cold_threshold = .168
df = df %>% ungroup() %>% 
    mutate(is_hot = ifelse(((woba_rolling > hot_threshold) & (pa_rolling>=min_pa_in_streak) & (game_num < 157)), 1, 0),
           is_cold = ifelse(((woba_rolling < cold_threshold) & (pa_rolling>=min_pa_in_streak) & (game_num < 157)), 1, 0))<jupyter_output><empty_output><jupyter_text>Calculate Streak, Post-Streak, and Expected Value Summary Stats<jupyter_code>df %>% ungroup() %>%
    filter(is_hot==1,
          !is.na(woba_rolling_future)) %>%
    #group_by(season) %>%
    mutate(weighted_woba_num=B_PA*season_woba_num,
          weighted_woba_denom=B_PA*season_woba_denom) %>%
    summarise(
        unique_hot_players = n_distinct(person.key),
        total_hot_streaks = sum(is_hot),
        total_hot_pa = sum(pa_rolling),
        mean_hot_woba = sum(woba_num_rolling, na.rm=TRUE)/sum(woba_denom_rolling, na.rm=TRUE),
        mean_post_hot_woba_nextgame = sum(woba_num_rolling_future_nextgame, na.rm=TRUE)/sum(woba_denom_rolling_future_nextgame, na.rm=TRUE),
        mean_post_hot_woba = sum(woba_num_rolling_future, na.rm=TRUE)/sum(woba_denom_rolling_future, na.rm=TRUE),
        expected_woba= mean(expected_woba, na.rm=TRUE),
        std_mean_post_hot_woba = 1.96 * (sd(woba_num_rolling_future/woba_denom_rolling_future, na.rm=TRUE) / sqrt(n())),
        upper_ci_post = mean_post_hot_woba + std_mean_post_hot_woba,
        lower_ci_post = mean_post_hot_woba - std_mean_post_hot_woba,
        mean_season_woba = sum(season_woba_num) / sum(season_woba_denom)
    )

df %>% ungroup() %>%
    filter(is_cold==1,
          !is.na(woba_rolling_future)) %>%
    mutate(weighted_woba_num=B_PA*season_woba_num,
          weighted_woba_denom=B_PA*season_woba_denom) %>%
    summarise(
        unique_hot_players = n_distinct(person.key),
        total_hot_streaks = sum(is_cold),
        total_hot_pa = sum(pa_rolling),
        mean_hot_woba = sum(woba_num_rolling, na.rm=TRUE)/sum(woba_denom_rolling, na.rm=TRUE),
        mean_post_hot_woba_nextgame = sum(woba_num_rolling_future_nextgame, na.rm=TRUE)/sum(woba_denom_rolling_future_nextgame, na.rm=TRUE),
        mean_post_hot_woba = sum(woba_num_rolling_future, na.rm=TRUE)/sum(woba_denom_rolling_future, na.rm=TRUE),
        expected_woba= mean(expected_woba, na.rm=TRUE),
        std_mean_post_hot_woba = 1.96 * (sd(woba_num_rolling_future/woba_denom_rolling_future, na.rm=TRUE) / sqrt(n())),
        upper_ci_post = mean_post_hot_woba + std_mean_post_hot_woba,
        lower_ci_post = mean_post_hot_woba - std_mean_post_hot_woba,
        mean_season_woba = sum(season_woba_num) / sum(season_woba_denom)
    )

# table of example streaks for the blog post
df %>% ungroup() %>%
    filter(is_hot==1,
           season==2018,
          !is.na(woba_rolling_future)) %>%
    select(person.key, game.date, pa_rolling, woba_rolling, woba_rolling_future, expected_woba) %>% 
    arrange(-woba_rolling) %>% head(10)

# table of example streaks for the blog post
df %>% ungroup() %>%
    filter(is_cold==1,
           season==2018,
          !is.na(woba_rolling_future)) %>%
    select(person.key, game.date, pa_rolling, woba_rolling, woba_rolling_future, expected_woba) %>% 
    arrange(woba_rolling) %>% head(10)
# t-test for significance
test = df %>% ungroup() %>%
    filter(is_hot==1,
          !is.na(woba_rolling_future)) %>%
    group_by(person.key,game.key) %>%
    summarise(
        total_hot_pa = sum(pa_rolling),
        post_hot_pa= sum(pa_rolling_future),
        mean_post_hot_woba = sum(woba_num_rolling_future, na.rm=TRUE)/sum(woba_denom_rolling_future, na.rm=TRUE),
        expected_woba= mean(expected_woba, na.rm=TRUE)
    )

test = test %>% filter(!is.na(mean_post_hot_woba),
                        post_hot_pa>=10)

t.test(test$mean_post_hot_woba, test$expected_woba)

test = df %>% ungroup() %>%
    filter(is_cold==1,
          !is.na(woba_rolling_future)) %>%
    group_by(person.key,game.key) %>%
    summarise(
        total_hot_pa = sum(pa_rolling),
        post_hot_pa= sum(pa_rolling_future),
        mean_post_hot_woba = sum(woba_num_rolling_future, na.rm=TRUE)/sum(woba_denom_rolling_future, na.rm=TRUE),
        expected_woba= mean(expected_woba, na.rm=TRUE)
    )

test = test %>% filter(!is.na(mean_post_hot_woba),
                        post_hot_pa>=10)
t.test(test$mean_post_hot_woba, test$expected_woba)
# regression for hot/cold effects
summary(lm(woba_rolling_future ~ expected_woba + is_hot + is_cold + site.key, 
           df %>% filter(!is.na(woba_rolling_future),
                        !is.na(is_hot))))
# inspect distributions of post-streak and expected woba
df %>% filter(is_hot==1) %>%
select(game.date, person.key, woba_rolling, 
       woba_rolling_future, expected_woba, 
       pa_rolling, pa_rolling_future,
      ) %>% 
select(woba_rolling_future) %>% ggplot(aes(x=woba_rolling_future)) + geom_histogram(bins=100)

df %>% filter(is_hot==1) %>%
select(game.date, person.key, woba_rolling, 
       woba_rolling_future, expected_woba, 
       pa_rolling, pa_rolling_future,
      ) %>% 
select(expected_woba) %>% ggplot(aes(x=expected_woba)) + geom_histogram(bins=100)<jupyter_output>Warning message:
“Removed 88 rows containing non-finite values (stat_bin).”<jupyter_text>Repeat the above steps for 7-game streaks<jupyter_code># take rolling sums of woba components to get a 5g woba
library(zoo)
num_games_streak = 7
num_games_post_streak = 7
min_pa_in_streak = 4 * num_games_streak #4
min_pa_post_streak = 3 * num_games_post_streak #0
# different offset used for post streak folling counts for 3 vs. 5 game configuration. paste this in if changing streak lenghts.
seven_game_post = list(c(1, 2, 3, 4, 5, 6, 7)) #list(c(1, 2, 3))
five_game_post = list(c(1, 2, 3, 4, 5))

# this is ugly indentation but makes it ahrder to forget to paste something when updating it
df = df %>%
         arrange(season, person.key, game.date, game.key) %>%
         group_by(season, person.key) %>%
         mutate(woba_num_rolling =   roll_sum(woba_numerator, num_games_streak, align = "right", fill = NA),
                pa_rolling =         roll_sum(B_PA, num_games_streak, align = "right", fill = NA),
                woba_denom_rolling = roll_sum(woba_denominator, num_games_streak, align = "right", fill = NA),
                woba_rolling = woba_num_rolling/woba_denom_rolling,
                woba_num_rolling_future =   rollapply(woba_numerator, FUN=sum, seven_game_post, align = "left", fill = NA),
                pa_rolling_future =         rollapply(B_PA, FUN=sum, seven_game_post, align = "left", fill = NA),
                woba_denom_rolling_future = rollapply(woba_denominator, FUN=sum,  seven_game_post, align = "left", fill = NA),
                woba_rolling_future = woba_num_rolling_future/woba_denom_rolling_future,
                
                woba_num_rolling_future_nextgame =   rollapply(woba_numerator, FUN=sum, list(c(1)), align = "left", fill = NA),
                woba_denom_rolling_future_nextgame  = rollapply(woba_denominator, FUN=sum,  list(c(1)), align = "left", fill = NA),
                woba_rolling_future_nextgame  = woba_num_rolling_future_nextgame /woba_denom_rolling_future_nextgame 
        ) 
# get seasons' quantiles for woba thresholds
#temp = df %>% group_by(season, person.key) %>% filter(sum(B_PA)>100) %>% ungroup()
df %>% 
    # only consider periods with at least 20 pa in them 
    filter(pa_rolling >= min_pa_in_streak, pa_rolling_future >= min_pa_post_streak) %>% 
    group_by(season, person.key, game.key) %>% summarise(woba = sum(woba_numerator) / sum(woba_denominator),
                                                  woba_rolling_avg = sum(woba_num_rolling, na.rm=TRUE)/sum(woba_denom_rolling, na.rm=TRUE),
                                                  pa = sum(B_PA)) %>%
    ungroup() %>%
    summarise(thresholds = quantile(woba_rolling_avg, probs = c(0.05, 0.95), na.rm=TRUE)) 
# create is hot/cold flags
hot_threshold = .532
cold_threshold = .194
df = df %>% ungroup() %>% 
    mutate(is_hot = ifelse(((woba_rolling > hot_threshold) & (pa_rolling>=min_pa_in_streak) & (game_num < 157)), 1, 0),
           is_cold = ifelse(((woba_rolling < cold_threshold) & (pa_rolling>=min_pa_in_streak) & (game_num < 157)), 1, 0))
df %>% ungroup() %>%
    filter(is_hot==1,
          !is.na(woba_rolling_future)) %>%
    summarise(
        unique_hot_players = n_distinct(person.key),
        total_hot_streaks = sum(is_hot),
        total_hot_pa = sum(pa_rolling),
        mean_hot_woba = sum(woba_num_rolling, na.rm=TRUE)/sum(woba_denom_rolling, na.rm=TRUE),
        mean_post_hot_woba = sum(woba_num_rolling_future, na.rm=TRUE)/sum(woba_denom_rolling_future, na.rm=TRUE),
        mean_post_hot_woba_nextgame = sum(woba_num_rolling_future_nextgame, na.rm=TRUE)/sum(woba_denom_rolling_future_nextgame, na.rm=TRUE),
        std_mean_post_hot_woba = 1.96 * (sd(woba_num_rolling_future/woba_denom_rolling_future, na.rm=TRUE) / sqrt(n())),
        upper_ci_post = mean_post_hot_woba + std_mean_post_hot_woba,
        lower_ci_post = mean_post_hot_woba - std_mean_post_hot_woba,
        expected_woba= mean(expected_woba, na.rm=TRUE)
    )

df %>% ungroup() %>%
    filter(is_cold==1,
          !is.na(woba_rolling_future)) %>%
    summarise(
        unique_hot_players = n_distinct(person.key),
        total_hot_streaks = sum(is_cold),
        total_hot_pa = sum(pa_rolling),
        mean_hot_woba = sum(woba_num_rolling, na.rm=TRUE)/sum(woba_denom_rolling, na.rm=TRUE),
        mean_post_hot_woba = sum(woba_num_rolling_future, na.rm=TRUE)/sum(woba_denom_rolling_future, na.rm=TRUE),
        mean_post_hot_woba_nextgame = sum(woba_num_rolling_future_nextgame, na.rm=TRUE)/sum(woba_denom_rolling_future_nextgame, na.rm=TRUE),
        std_mean_post_hot_woba = 1.96 * (sd(woba_num_rolling_future/woba_denom_rolling_future, na.rm=TRUE) / sqrt(n())),
        upper_ci_post = mean_post_hot_woba + std_mean_post_hot_woba,
        lower_ci_post = mean_post_hot_woba - std_mean_post_hot_woba,
        expected_woba= mean(expected_woba, na.rm=TRUE)
    )<jupyter_output><empty_output><jupyter_text>test = df %>% ungroup() %>%    filter(is_hot==1,          !is.na(woba_rolling_future)) %>%    group_by(person.key,game.key) %>%    summarise(        total_hot_pa = sum(pa_rolling),        post_hot_pa= sum(pa_rolling_future),        mean_post_hot_woba = sum(woba_num_rolling_future, na.rm=TRUE)/sum(woba_denom_rolling_future, na.rm=TRUE),        expected_woba= mean(expected_woba, na.rm=TRUE)    )test = test %>% filter(!is.na(mean_post_hot_woba),                        post_hot_pa>=10)t.test(test$mean_post_hot_woba, test$expected_woba)test = df %>% ungroup() %>%    filter(is_cold==1,          !is.na(woba_rolling_future)) %>%    group_by(person.key,game.key) %>%    summarise(        total_hot_pa = sum(pa_rolling),        post_hot_pa= sum(pa_rolling_future),        mean_post_hot_woba = sum(woba_num_rolling_future, na.rm=TRUE)/sum(woba_denom_rolling_future, na.rm=TRUE),        expected_woba= mean(expected_woba, na.rm=TRUE)    )test = test %>% filter(!is.na(mean_post_hot_woba),                        post_hot_pa>=10)t.test(test$mean_post_hot_woba, test$expected_woba)<jupyter_code>options("scipen"=100, "digits"=3)
summary(lm(woba_rolling_future ~ expected_woba + is_hot + is_cold + site.key, 
           df %>% filter(!is.na(woba_rolling_future),
                        !is.na(is_hot), season_pa>50)))<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11720742630049232, 'fraction_numerical': 0.015540660774660419, 'mean_word_length': 3.5840130505709626, 'pattern_counts': {'":': 0, '<': 28, '<?xml version=': 0, '>': 124, 'https://': 1, 'lorem ipsum': 0, 'www.': 1, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 10, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '170520', 'license': 'MIT', 'n_tokens_mistral': 6358, 'n_tokens_neox': 5990, 'n_words': 1100, 'path': 'hot_streaks/hot-streaks-park-adjusted.ipynb', 'repo_name': 'jldbc/Saber', 'size': '16859'}