<jupyter_start><jupyter_text>Make standardized schedule formats.<jupyter_code>import pandas as pd
df = pd.read_csv("schedule21.csv")
df.loc[df["schedule_date"].str.match("1/.*"),"schedule_date"] = df.loc[df["schedule_date"].str.match("1/.*"),"schedule_date"].map(lambda s: s[:-1]+"2")
df.rename({"pr_spread":"pr_score_home","pr_total":"pr_score_away"},axis=1,inplace=True)
df.drop("pr_hfa",axis=1,inplace=True)
df]
df.to_csv("schedule21.csv",index=False)
df
df["schedule_date"].where(df["schedule_date"].map(lambda s: s[:2] == "1/"))
df.loc[df["schedule_date"].str.match("1/.*"),"schedule_date"] = df.loc[df["schedule_date"].str.match("1/.*"),"schedule_date"].map(lambda s: s[:-1]+"2")
df
import pandas as pd
import numpy as np
def to_id(name):
    try:
        return df_teams["team_id"][df_teams["team_name"] == name].item()
    except:
        return name

def to_div(id):
    return df_teams["team_division"][df_teams["team_id"] == id].dropna().iloc[0]

def to_conf(id):
    return to_div(id)[:3]
df_all = pd.read_csv("spreadspoke_scores.csv",usecols=["schedule_date","schedule_season","schedule_week","schedule_playoff","team_home","score_home","score_away","team_away","team_favorite_id","spread_favorite","over_under_line","stadium_neutral"])
df_teams = pd.read_csv("nfl_teams.csv")
def make_csv(year):
    df = df_all.query("schedule_season == @year").copy()
    df.reset_index(inplace=True,drop=True)
    if df.loc[0,"schedule_season"] == 2021:
        df.loc[df["schedule_date"].map(lambda s: s[:2] == "1/")
               ,"schedule_date"] = df.loc[df["schedule_date"].map(lambda s: s[:2] == "1/"),
                                          "schedule_date"].map(lambda s: s[:-1]+"2")
        return df[["schedule_date","schedule_season","schedule_week","schedule_playoff",
    df.loc[:,["team_home","team_away"]] = df[["team_home","team_away"]].applymap(to_id).copy()
    df["over_under_line"] = pd.to_numeric(df["over_under_line"])
    df.loc[df["team_favorite_id"] == df["team_home"],"spread_favorite"] *= -1
    df.loc[:,["pr_score_home","pr_score_away","in_div","in_conf"]] = np.nan
    df["in_div"] = (df["team_home"].map(to_div) == df["team_away"].map(to_div))
    df["in_conf"] = (df["team_home"].map(to_conf) == df["team_away"].map(to_conf))
    return df
make_csv(2021)
df = make_csv(2021)
df.loc[df["schedule_date"].map(lambda s: s[:2] == "1/"),"schedule_date"] = df.loc[df["schedule_date"].map(lambda s: s[:2] == "1/"),"schedule_date"].map(lambda s: s[:-1]+"2")
df
mean_score = np.mean(list(df["score_home"]) + list(df["score_away"]))
pr_offense = {}
pr_defense = {}
teams = sorted(list(set(df["team_home"])))
df_scores = pd.DataFrame(index=range(2*len(df)),columns = [t+"_Off" for t in teams] + [t+"_Def" for t in teams]+["HFA"]+["Score"])
for i in range(len(df)):
    row = df.iloc[i]
    sf = row["spread_favorite"]
    ou = row["over_under_line"]
    ht = row["team_home"]
    at = row["team_away"]
    df_scores.iloc[2*i].loc[ht+"_Off"] = 1
    df_scores.iloc[2*i].loc[at+"_Def"] = -1
    df_scores.iloc[2*i].loc["Score"] = (ou+sf)/2
    df_scores.iloc[2*i].loc["HFA"] = 0.5
    df_scores.iloc[2*i+1].loc[at+"_Off"] = 1
    df_scores.iloc[2*i+1].loc[ht+"_Def"] = -1
    df_scores.iloc[2*i+1].loc["Score"] = (ou-sf)/2
    df_scores.iloc[2*i+1].loc["HFA"] = -0.5
df_scores["Score"] = df_scores["Score"] - mean_score
df_scores = df_scores.fillna(0)
df_scores
reg = LinearRegression(fit_intercept=False).fit(df_scores.drop("Score",axis=1),df_scores["Score"])
reg.coef_
dict(zip(df_scores.columns,reg.coef_))
for year in range(2016,2022):
    df = make_csv(year)
    df.to_csv(f"schedule{year}.csv",index=False)
df21.to_csv("schedule21.csv",index=False)
teams = sorted(list(set(df21["team_home"])))
pd.Series({t:to_div(t) for t in teams}).to_csv("divisions.csv")
import anvil.server

anvil.server.connect("3KLT2ZIMIW24YNI7NGK7LDOI-OFIY2TEEA7UANSRQ")
tol = .0001



def to_div(id):
    return df_teams["team_division"][df_teams["team_id"] == id].dropna().iloc[0]

def to_conf(id):
    return to_div(id)[:3]

def pr_spread(row):
    return pr_dict[row["team_home"]] - pr_dict[row["team_away"]]+pr_dict['HFA']

reverse_dict = {'Win':'Loss','Loss':'Win', 'Tie':'Tie'}

def get_score(row):
    total = row["pr_total"] + 10*np.random.randn()
    diff = row["pr_spread"] + 14*np.random.randn()
    ps = round(0.5*(total + diff))
    pa = round(0.5*(total - diff))
    # Artificially get rid of most ties.
    if ps == pa:
        x = np.random.randn()
        if (x > -1.6) and (x < 0):
            pa = pa+3
        elif x >= 0:
            ps = ps+3
    return (ps, pa)

def get_outcome(row):
    s = row["Points_scored"]
    a = row["Points_allowed"]
    if s > a:
        return "Win"
    elif s < a:
        return "Loss"
    elif s == a:
        return "Tie"

def get_WLT(games):
    s = games.Outcome.value_counts()
    return (s.get("Win",0) + 0.5*s.get("Tie",0))/len(games)

def get_strength(teams):
    return get_WLT(df21_ind[df21_ind["Team"].isin(teams)])

def get_victories(team):
    return set(df21_ind["Opponent"][(df21_ind["Team"] == team) & (df21_ind["Outcome"] == "Win")])

def get_opps(team):
    return set(df21_ind["Opponent"][df21_ind["Team"] == team])

def get_common(teams):
    opps = []
    for t in teams:
        opps.append(get_opps(t))
    return set.intersection(*opps)
# Haven't thought about a length 4 dictionary.
def analyze_dict(d):
    tol = .0001
    outs = sorted(d.values(),reverse=True)
    if isclose(outs[0],outs[-1],abs_tol=tol):
        return None
    teams = [k for k in d.keys() if isclose(d[k],outs[0],abs_tol = tol)]
    if len(teams) == 1:
        return teams[0]
    if to_div(teams[0]) == to_div(teams[1]):
        return break_tie_div(teams)
    else:
        return break_tie_conf(teams)
break_tie_fns = {}
# Are there any numerical precision problems here?
def fd21(teams):
    wlt_dict = {teams[0]:get_WLT(df21_ind[(df21_ind["Team"] == teams[0]) & (df21_ind["Opponent"] == teams[1])])}
    wlt_dict[teams[1]] = get_WLT(df21_ind[(df21_ind["Team"] == teams[1]) & (df21_ind["Opponent"] == teams[0])])
    return analyze_dict(wlt_dict)

break_tie_fns[("div",2,1)] = fd21
def fd22(teams):
    div_dict = {t: get_WLT(df21_ind[(df21_ind.Team == t) & (df21_ind.in_div)]) for t in teams}
    return analyze_dict(div_dict)

break_tie_fns[("div",2,2)] = fd22
def fd23(teams):
    common = get_common(teams)
    common_dict = {t: get_WLT(df21_ind[(df21_ind.Team == t) & (df21_ind.Opponent.isin(common))]) for t in teams}
    return analyze_dict(common_dict)

break_tie_fns[("div",2,3)] = fd23
def fd24(teams):
    conf_dict = {t: get_WLT(df21_ind[(df21_ind.Team == t) & (df21_ind.in_conf)]) for t in teams}
    return analyze_dict(conf_dict)

break_tie_fns[("div",2,4)] = fd24
def fd25(teams):
    wlt_dict = {t: get_strength(get_victories(t)) for t in teams}
    return analyze_dict(wlt_dict)

break_tie_fns[("div",2,5)] = fd25
break_tie_fns[("div",3,5)] = fd25
def fd26(teams):
    wlt_dict = {t: get_strength(get_opps(t)) for t in teams}
    return analyze_dict(wlt_dict)

break_tie_fns[("div",2,6)] = fd26
break_tie_fns[("div",3,6)] = fd26
# The negatives are because we want "lower rank" to correspond to higher numbers
# Maybe would be better to have a reverse flag in the dictionary
def fd27(teams):
    conf = to_conf(teams[0])
    ps = list(df_standings.query("Conference==@conf").sort_values("Points scored",ascending=False).index)
    pa = list(df_standings.query("Conference==@conf").sort_values("Points allowed",ascending=True).index)
    rank_dict = {t: -ps.index(t)-pa.index(t) for t in teams}
    return analyze_dict(rank_dict)

break_tie_fns[("div",2,7)] = fd27
break_tie_fns[("div",3,7)] = fd27
# The negatives are because we want "lower rank" to correspond to higher numbers
def fd28(teams):
    ps = list(df_standings.sort_values("Points scored",ascending=False).index)
    pa = list(df_standings.sort_values("Points allowed",ascending=True).index)
    rank_dict = {t: -ps.index(t)-pa.index(t) for t in teams}
    return analyze_dict(rank_dict)

break_tie_fns[("div",2,8)] = fd28
break_tie_fns[("div",3,8)] = fd28
def fd29(teams):
    common = get_common(teams)
    pt_dict = {}
    for t in teams:
        df_common = df21_ind[["Points_scored","Points_allowed"]][(df21_ind.Team == t) & (df21_ind.Opponent.isin(common))].sum()
        pt_dict[t] = df_common["Points_scored"]-df_common["Points_allowed"]
    return analyze_dict(pt_dict)

break_tie_fns[("div",2,9)] = fd29
break_tie_fns[("div",3,9)] = fd29
# Would it be better to use df_standings for this?
def fd210(teams):
    pt_dict = {}
    for t in teams:
        df_common = df21_ind[["Points_scored","Points_allowed"]][df21_ind.Team == t].sum()
        pt_dict[t] = df_common["Points_scored"]-df_common["Points_allowed"]
    return analyze_dict(pt_dict)

break_tie_fns[("div",2,10)] = fd210
break_tie_fns[("div",3,10)] = fd210
# How does the coin toss work for three teams?
def fd212(teams):
    return np.random.choice(teams)

break_tie_fns[("div",2,12)] = fd212
break_tie_fns[("div",3,12)] = fd212
def fd31(teams):
    df = df21_ind[df21_ind["Team"].isin(teams) & df21_ind["Opponent"].isin(teams)]
    wlt_dict = {t: get_WLT(df[df.Team == t]) for t in teams}
    return analyze_dict(wlt_dict)

break_tie_fns[("div",3,1)] = fd31
def fd32(teams):
    wlt_dict = {t: get_WLT(df21_ind[(df21_ind.Team == t) & (df21_ind.in_div)]) for t in teams}
    return analyze_dict(wlt_dict)

break_tie_fns[("div",3,2)] = fd32
def fd33(teams):
    opps = []
    for t in teams:
        opps.append(get_opps(t))
    common = set.intersection(*opps)
    wlt_dict = {t: get_WLT(df21_ind[(df21_ind.Team == t) & (df21_ind.Opponent.isin(common))]) for t in teams}
    return analyze_dict(wlt_dict)

break_tie_fns[("div",3,3)] = fd33
def fd34(teams):
    wlt_dict = {t: get_WLT(df21_ind[(df21_ind.Team == t) & (df21_ind.in_conf)]) for t in teams}
    return analyze_dict(wlt_dict)

break_tie_fns[("div",3,4)] = fd34
def fc21(teams):
    df_head = df21_ind[(df21_ind["Team"] == teams[0]) & (df21_ind["Opponent"] == teams[1])]
    if len(df_head) == 0:
        return None
    wlt_dict = {teams[0]:get_WLT(df21_ind[(df21_ind["Team"] == teams[0]) & (df21_ind["Opponent"] == teams[1])])}
    wlt_dict[teams[1]] = get_WLT(df21_ind[(df21_ind["Team"] == teams[1]) & (df21_ind["Opponent"] == teams[0])])
    return analyze_dict(wlt_dict)

break_tie_fns[("conf",2,1)] = fc21
def fc23(teams):
    common = get_common(teams)
    if len(common) < 4:
        return None
    common_dict = {t: get_WLT(df21_ind[(df21_ind.Team == t) & (df21_ind.Opponent.isin(common))]) for t in teams}
    return analyze_dict(common_dict)

break_tie_fns[("conf",2,3)] = fc23
def fc28(teams):
    pt_dict = {}
    for t in teams:
        df_conf = df21_ind[["Points_scored","Points_allowed"]][(df21_ind.Team == t) & (df21_ind.in_conf)].sum()
        pt_dict[t] = df_conf["Points_scored"]-df_conf["Points_allowed"]
    return analyze_dict(pt_dict)

break_tie_fns[("conf",2,8)] = fc28
break_tie_fns[("conf",2,2)] = break_tie_fns[("div",2,4)]
break_tie_fns[("conf",2,4)] = break_tie_fns[("div",2,5)]
break_tie_fns[("conf",2,5)] = break_tie_fns[("div",2,6)]
break_tie_fns[("conf",2,6)] = break_tie_fns[("div",2,7)]
break_tie_fns[("conf",2,7)] = break_tie_fns[("div",2,8)]
break_tie_fns[("conf",2,9)] = break_tie_fns[("div",2,10)]
break_tie_fns[("conf",2,11)] = break_tie_fns[("div",2,12)]
def find_sweep(teams):
    for t in teams:
        others = [x for x in teams if x != t]
        outcomes = []
        temp_df = df21_ind[(df21_ind["Team"] == t) & (df21_ind["Opponent"].isin(others))]
        outcomes += list(temp_df["Outcome"])
        if len(outcomes) >= len(others): # make sure teams played
            if set(outcomes) == {'Win'}:
                return t
            elif set(outcomes) == {'Loss'}:
                return break_tie_conf(others)
    return None
break_tie_fns[("conf",3,2)] = find_sweep
break_tie_fns[("conf",3,3)] = break_tie_fns[("div",3,4)]
break_tie_fns[("conf",3,4)] = break_tie_fns[("conf",2,3)]
break_tie_fns[("conf",3,5)] = break_tie_fns[("div",2,5)]
break_tie_fns[("conf",3,6)] = break_tie_fns[("div",2,6)]
break_tie_fns[("conf",3,7)] = break_tie_fns[("div",2,7)]
break_tie_fns[("conf",3,8)] = break_tie_fns[("div",2,8)]
break_tie_fns[("conf",3,9)] = break_tie_fns[("conf",2,8)]
break_tie_fns[("conf",3,10)] = break_tie_fns[("div",2,10)]
break_tie_fns[("conf",3,12)] = break_tie_fns[("div",2,12)]
def break_tie_div(teams):
    scenario = 2 if len(teams) == 2 else 3
    rules = sorted([c for (a,b,c) in break_tie_fns.keys() if a == 'div' and b == scenario])
    for c in rules:
        t = break_tie_fns[("div",scenario,c)](teams)
        if t is not None:
            return t
def break_tie_conf(teams):
    scenario = 2 if len(teams) == 2 else 3
    rules = sorted([c for (a,b,c) in break_tie_fns.keys() if a == 'conf' and b == scenario])
    for c in rules:
        t = break_tie_fns[("conf",scenario,c)](teams)
        if t is not None:
            return t
# Any numerical precision problems here?
def get_div_winners():
    winner_dict = {}
    divs = sorted(list(set(df_standings.Division)))
    for div in divs:
        df = df_standings[df_standings["Division"] == div].sort_values("WLT",ascending=False).copy()
        t = analyze_dict(dict(zip(df.index,df.WLT)))
        if t is None:
            t = break_tie_div(list(df.index))
        winner_dict[div] = t
    return winner_dict
def rank_div_winners():
    dw = get_div_winners()
    playoffs = {}
    for conf in ["AFC","NFC"]:
        playoffs[conf] = []
        teams = [dw[x] for x in dw.keys() if x[:3] == conf]
        while len(playoffs[conf]) < 3:
            df = df_standings[df_standings["Team"].isin(teams)].sort_values("WLT",ascending=False).copy()
            t = analyze_dict(dict(df.loc[teams,"WLT"]))
            if t is None:
                t = break_tie_conf(teams)
            playoffs[conf].append(t)
            try:
                teams.remove(t)
            except:
                pass
        playoffs[conf].append(teams[0])
    return playoffs
# If I understand correctly, this should not include the division winner.
# Any numerical precision problems here?
def rank_within_divs():
    dw = get_div_winners()
    dr = {}
    for div in dw.keys():
        dr[div] = []
        df = df_standings[(df_standings["Division"] == div) & ~(df_standings["Team"] == dw[div])].sort_values("WLT",ascending=False).copy()
        teams = list(df.index)
        while len(dr[div]) < 2:
            t = analyze_dict(dict(df.loc[teams,"WLT"]))
            if t is None:
                t = break_tie_div(teams)
            dr[div].append(t)
            teams.remove(t)
        dr[div].append(teams[0])
    return dr
df = pd.read_csv("spreadspoke_scores.csv")
df_teams = pd.read_csv("nfl_teams.csv")
df21 = df.query('schedule_season==2021').copy()
series_pr = pd.read_csv("power_ratings.csv",index_col=0,squeeze=True)
pr_dict = series_pr.to_dict()
df21[["team_home","team_away"]] = df21[["team_home","team_away"]].applymap(to_id)
teams = set(df21["team_home"])
df21["pr_spread"] = df21[["team_home","team_away"]].apply(pr_spread,axis=1)
df = pd.read_csv("spreadspoke_scores.csv")
df_teams = pd.read_csv("nfl_teams.csv")
df21 = df.query('schedule_season==2021').copy()
series_pr = pd.read_csv("power_ratings.csv",index_col=0,squeeze=True)
pr_dict = series_pr.to_dict()
df21[["team_home","team_away"]] = df21[["team_home","team_away"]].applymap(to_id)
teams = set(df21["team_home"])
df21["pr_spread"] = df21[["team_home","team_away"]].apply(pr_spread,axis=1)
df21["div_home"] = df21["team_home"].map(to_div)
df21["div_away"] = df21["team_away"].map(to_div)
df21["in_div"] = (df21["div_home"] == df21["div_away"])
df21["in_conf"] = (df21["div_home"].map(lambda s: s[:3]) == df21["div_away"].map(lambda s: s[:3]))
df_standings = pd.DataFrame(index=sorted(teams,key=to_div),columns=["Team","Wins","Losses","Ties","Points_scored","Points_allowed","WLT","Division","Conference"])
df_standings.Team = df_standings.index
df_standings.Division = df_standings.index.map(to_div)
df_standings.Conference = df_standings.index.map(to_conf)
spreads = df21["pr_spread"].to_numpy()
df21_ind = pd.DataFrame(index=range(2*len(df21)),columns=["Team","Opponent","Points_scored","Points_allowed","Outcome","in_div","in_conf","pr_spread","pr_total"])
@anvil.server.callable
def simulate_season(reps):
    assert reps < 5000
    df21_ind.loc[::2,"pr_spread"] = spreads
    df21_ind.loc[1::2,"pr_spread"] = -spreads
    df21_ind["pr_total"] = 40
    cols = list(df21_ind.columns)
    inds = [cols.index("Team"),cols.index("Opponent"),cols.index("in_div"),cols.index("in_conf")]
    df21_ind.iloc[::2,inds] = df21[["team_home","team_away","in_div","in_conf"]]
    df21_ind.iloc[1::2,inds] = df21[["team_away","team_home","in_div","in_conf"]]
    playoff_dict = {t:0 for t in sorted(teams)}
    for i in range(reps):
        scores = df21_ind.loc[::2].apply(get_score,axis=1).to_list()
        df21_ind.loc[::2,["Points_scored","Points_allowed"]] = scores
        df21_ind.loc[1::2,["Points_allowed","Points_scored"]] = scores
        df21_ind.Outcome = df21_ind.apply(get_outcome,axis=1)

        for a,b in df21_ind.groupby("Team"):
            res = b.Outcome.value_counts()
            df_standings.loc[a,["Wins","Losses","Ties","Points_scored","Points_allowed"]] = [res.get("Win",0), res.get("Loss",0), res.get("Tie",0),
                b["Points_scored"].sum(),b["Points_allowed"].sum()]
        df_standings["WLT"] = (df_standings["Wins"]+0.5*df_standings["Ties"])/(df_standings["Wins"]+df_standings["Ties"]+df_standings["Losses"])

        dw = get_div_winners()
        dw_ranked = rank_div_winners()
        dr = rank_within_divs()
        wild_cards = {}
        for conf in ["AFC","NFC"]:
            wild_cards[conf] = []
            while len(wild_cards[conf]) < 3:
                top_teams = [dr[x][0] for x in dr.keys() if x[:3] == conf]
                t = analyze_dict(dict(df_standings.loc[top_teams,"WLT"]))
                if t is None:
                    t = break_tie_conf(top_teams)
                wild_cards[conf].append(t)
                try:
                    dr[to_div(t)].remove(t)
                except:
                    pass
        playoffs = {conf:dw_ranked[conf] + wild_cards[conf] for conf in ["AFC","NFC"]}
        for v in playoffs.values():
            for x in v:
                playoff_dict[x] += 1
    return {k:round(v/reps,3) for k,v in playoff_dict.items()}<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.18880363125472818, 'fraction_numerical': 0.03296228250297201, 'mean_word_length': 4.299828178694158, 'pattern_counts': {'":': 2, '<': 12, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '244617', 'license': 'BSD-3-Clause', 'n_tokens_mistral': 7744, 'n_tokens_neox': 7326, 'n_words': 1154, 'path': 'schedules/.ipynb_checkpoints/make_schedule-checkpoint.ipynb', 'repo_name': 'ChristopherDavisUCI/NFL2021-Simulation', 'size': '18506'}