<jupyter_start><jupyter_text>European Club data - Analysis of goalsWritten report for this analysis can be found [here](../reports/club_01_goals_around_europe.md)<jupyter_code>## our packaged code
from src import utilities
## suppress warnings
import warnings
warnings.filterwarnings('ignore')
from scipy.stats import norm  
import numpy as np
import pandas as pd
## visualisation
%matplotlib inline
import matplotlib.pyplot as plt
plt.style.use("seaborn-whitegrid")

import seaborn as sns
# sns.set()
sns.set(rc={'figure.figsize':(10, 5)})

# from mpl_toolkits.basemap import Basemap
df = utilities.get_master("fulldata")

df = df[df.HomeAway == "Home"]

df.dropna(subset=['TotalGoals'], inplace=True)

df['Date'] = pd.to_datetime(df['Date'], format="%Y-%m-%d")
df["Month"] = df["Date"].apply(lambda x: x.strftime("%m"))
df["Day_Of_Week"] = df["Date"].apply(lambda x: x.strftime("%w"))

attrib_cols = ["Date", "Month", "Day_Of_Week", "HomeAway", "Season", "Country", "Tier", "Team", "TeamOpp", "Manager", "ManagerOpp", "Referee", 
               "Stadium", "Latitude", "Longitude"]
metric_cols = ["TotalGoals"]

df = df[attrib_cols + metric_cols]

df.shape
df.describe(include="all").T.fillna("")<jupyter_output><empty_output><jupyter_text>Overall Trends<jupyter_code>mean = df.TotalGoals.mean()
std = df.TotalGoals.std()
low = mean - std
high = mean + std

mean, std, low, high
df.TotalGoals.plot(kind='hist', density=True, bins=range(int(df.TotalGoals.max()+1)))

xmin, xmax = plt.xlim()
x = np.linspace(xmin, xmax, 100)
p = norm.pdf(x, mean, std)
plt.plot(x, p, 'k', linewidth=2)
# title = "Fit results: mu = %.2f,  std = %.2f" % (mean, std)
# plt.title(title)

plt.axvline(x=mean, linewidth=3, color='g', linestyle="--", label="mu")

plt.savefig("../reports/figures/club_01_hist.PNG")
plt.show()
for col in ["Season", "Month", "Day_Of_Week", "Country", "Tier"]:
    print("\n#######\n")
    GpG = df[[col, "TotalGoals"]].groupby(col)
    print(GpG.describe(include="all"))
#     df[[col, "TotalGoals"]].boxplot(by=col, showmeans=True, widths=0.7, figsize=(10,5)).get_figure().gca().set_title("")
    sns.boxplot(x=col, y="TotalGoals", data=df.sort_values(by=col), showmeans=True, width=0.6)
    plt.xticks(rotation=90)
    
    plt.savefig("../reports/figures/club_01_boxplot_{0}.PNG".format(col))
    plt.show()<jupyter_output>#######

          TotalGoals                                              
               count      mean       std  min  25%  50%  75%   max
Season                                                            
1993-1994     3346.0  2.643455  1.699524  0.0  1.0  2.0  4.0  11.0
1994-1995     4617.0  2.690492  1.718282  0.0  1.0  2.0  4.0  10.0
1995-1996     5103.0  2.658828  1.694471  0.0  1.0  2.0  4.0  11.0
1996-1997     5981.0  2.587862  1.702795  0.0  1.0  2.0  4.0  13.0
1997-1998     6733.0  2.624981  1.725868  0.0  1.0  2.0  4.0  11.0
1998-1999     6531.0  2.629613  1.682949  0.0  1.0  2.0  4.0  10.0
1999-2000     6532.0  2.657226  1.701303  0.0  1.0  2.0  4.0  12.0
2000-2001     6553.0  2.684725  1.694666  0.0  1.0  2.0  4.0  10.0
2001-2002     6473.0  2.662753  1.697212  0.0  1.0  2.0  4.0  11.0
2002-2003     4391.0  2.599408  1.655140  0.0  1.0  2.0  4.0  10.0
2003-2004     5267.0  2.644959  1.670326  0.0  1.0  2.0  4.0  10.0
2004-2005     5644.0  2.576187  1.665243  0.0  1.0  [...]<jupyter_text>Most and Fewest Goals<jupyter_code>min_games = 30
top_n = 10

tops_list = []

for col in ["Team", "TeamOpp", "Manager", "ManagerOpp", "Stadium", "Referee"]:
    print("\n#######\n")
    for txt, asc, hilo in [("Top", False, high), ("Bottom", True, low)]:
        print("{0} {1} average goals by {2} (minimum {3} matches)...".format(txt, top_n, col, min_games))
        top = df[df[col].isin(df[col].value_counts()[df[col].value_counts() > min_games].index)].groupby(col).TotalGoals
        top = pd.DataFrame(top.mean().sort_values(ascending=asc).head(top_n))
        top["Variable"] = col
        tops_list.append(top)
        print(top)
        top.plot(kind="bar", figsize=(10,5))
        plt.xticks(rotation=90)
        plt.axhline(y=mean, linewidth=3, color='k', linestyle="--", label="Overall mean")
        plt.axhline(y=hilo, linewidth=3, color='r', linestyle="--", label="1 stdev from mean")
        plt.legend(bbox_to_anchor=(1.1, 1.05))
        plt.show()
    
tops = pd.concat(tops_list, axis=0) #.sort_values(by="TotalGoals", ascending=False).head(30)
tops["Value (Variable)"] = tops.index + " (" + tops.Variable + ")"
tops.set_index("Value (Variable)", inplace=True)
tops.drop(columns="Variable", inplace=True)
# tops

# tops = pd.concat(tops_list, axis=0) #.sort_values(by="TotalGoals", ascending=False).head(30)
top_n = 30

for txt, asc, hilo in [("Top", False, high), ("Bottom", True, low)]:
#     print("{0} {1} average goals by {2} (minimum {3} matches)...".format(txt, top_n, col, min_games))
#     top = df[df[col].isin(df[col].value_counts()[df[col].value_counts() > min_games].index)].groupby(col).TotalGoals
#     top = pd.DataFrame(top.mean().sort_values(ascending=asc).head(top_n))
#     top["Variable"] = col
#     tops_list.append(top)
    tmp = tops.sort_values(by="TotalGoals", ascending=asc).head(top_n)
    print(tmp)
    tmp.plot(kind="bar", figsize=(10,5))
    plt.xticks(rotation=90)
    plt.axhline(y=mean, linewidth=3, color='k', linestyle="--", label="Overall mean")
    plt.axhline(y=hilo, linewidth=3, color='r', linestyle="--", label="1 stdev from mean")
    plt.legend(bbox_to_anchor=(1.1, 1.05))
    
    plt.savefig("../reports/figures/club_01_bar_{0}.PNG".format(txt), bbox_inches='tight')
    plt.show()
    
atts = ["Country", "Tier"]
for col in ["Manager", "ManagerOpp", "Stadium", "Referee", "Latitude", "Longitude"]:
#     for att in ["Country", "Tier"]:
    print("\n#######\n")
    print("Sample size and means with {0} by {1}".format(col, ", ".join(atts)))
    print(df.loc[pd.notnull(df[col]), ].groupby(atts).TotalGoals.agg(["size", "mean"]).sort_index())<jupyter_output>#######

Sample size and means with Manager by Country, Tier
               size      mean
Country Tier                 
England 1     11215  2.619795
        2      9467  2.576423
        3      2379  2.550231
        4       544  2.591912

#######

Sample size and means with ManagerOpp by Country, Tier
               size      mean
Country Tier                 
England 1     11206  2.621542
        2      9486  2.591292
        3      2371  2.639814
        4       544  2.531250

#######

Sample size and means with Stadium by Country, Tier
                size      mean
Country  Tier                 
England  1     18512  2.630672
         2     20698  2.576771
         3     10430  2.591467
         4      7057  2.574607
         5      1005  2.766169
France   1      7804  2.421451
         2      3120  2.339423
Germany  1      6728  2.918549
         2      2439  2.717097
Scotland 1      5351  2.666417
         2      2820  2.706383
         3      1657  2.939650
         4      1[...]<jupyter_text>Mapping Goals<jupyter_code># mapdata=df.dropna(subset=['Latitude', 'Longitude'])
mapdata = df.dropna(subset=['Latitude', 'Longitude']).groupby(['Latitude', 'Longitude', 'Country']).TotalGoals\
    .mean().reset_index()

# fg = sns.FacetGrid(data=mapdata, hue='Country', height=6, aspect=.9)
# fg.map(plt.scatter, 'Longitude', 'Latitude').add_legend()

# sns.lmplot(x='Longitude', y='Latitude', s='TotalGoals', hue='Country', data=mapdata, fit_reg=False, 
#            x_jitter=0.1, y_jitter=0.1, markers="o", palette="viridis", height=7)

sns.relplot(x="Longitude", y="Latitude", hue="Country", size="TotalGoals",
            sizes=(100, 400), alpha=.5, palette="muted", aspect=1.2,
            height=7, data=mapdata)

plt.savefig("../reports/figures/club_01_map.PNG")
# # Extract the data we're interested in
# lat = df.groupby(['Stadium'])[["Latitude"]].max().values
# lon = df.groupby(['Stadium'])[["Longitude"]].max().values

# # print lat, lon

# # 1. Draw the map background
# fig = plt.figure(figsize=(8, 8))
# m = Basemap(projection='lcc', resolution='h', 
#             lat_0=46., lon_0=-3.,
#             width=3E6, height=3E6)
# m.shadedrelief()
# m.drawcoastlines(color='gray')
# m.drawcountries(color='black')

# # 2. scatter stadium data
# m.scatter(lon, lat, latlon=True,
#           color='r', s=40,
#           alpha=0.5)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.17754662840746055, 'fraction_numerical': 0.09732185557149689, 'mean_word_length': 2.989031950405341, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 5, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '615159', 'license': 'MIT', 'n_tokens_mistral': 3672, 'n_tokens_neox': 3239, 'n_words': 655, 'path': 'notebooks/club_01_goals_around_europe.ipynb', 'repo_name': 'deacona/the-ball-is-round', 'size': '8364'}