<jupyter_start><jupyter_text>This is using [the tutorial for the TopDownHockey_Scraper](https://github.com/TopDownHockey/TopDownHockey_Scraper/blob/main/Python%20Hockey%20Analytics%20Tutorial.ipynb).<jupyter_code>import matplotlib.pyplot as plt
import numpy as np
import pandas as pd
import seaborn as sns
import TopDownHockey_Scraper.TopDownHockey_EliteProspects_Scraper as tdhepscrape
%%time

# These are both lists, and each of them contain two strings. We use the equals sign to assign these lists to a named object which gets stored and can be used later.

leagues = ["ahl", "nhl"]
seasons = ["2017-2018", "2018-2019"]

# The get_skaters command not only returns a dataframe once it is completed, but also prints out messages as it runs in order to keep you updated on its progress.

ahl_nhl_skaters_1719_original = tdhepscrape.get_skaters(leagues, seasons)
# To view an object, simply enter its name and then run the cell.

ahl_nhl_skaters_1719_original
# Note to self, running a section of code that changes a variable changes that variable for everything

ahl_nhl_skaters_1719 = ahl_nhl_skaters_1719_original.drop(columns = 'player')
ahl_nhl_skaters_1719
# loc syntax: dataframe.loc[dataframe.column==value_to_match]

ahl_nhl_skaters_1719.loc[ahl_nhl_skaters_1719.playername=="<NAME>"]
ahl_nhl_skaters_1719.playername.iloc[0]
ahl_nhl_skaters_1719.loc[ahl_nhl_skaters_1719.playername=="<NAME>"]
#str.strip() syntax: dataframe.column.str.strip()

ahl_nhl_skaters_1719.playername = ahl_nhl_skaters_1719.playername.str.strip()

# We can use loc to filter out locate rows and columns that meet multiple conditions. 
# We use parentheses to specify each case and use | to pass the first OR second condition. 

ahl_nhl_skaters_1719.loc[(ahl_nhl_skaters_1719.playername=="<NAME>") | (ahl_nhl_skaters_1719.playername=="<NAME>")]
# Here we use loc to filter out columns with a certain name. We use :, then pass a list of column names in brackets to locate columns rather than rows.

# loc syntax for locating by column name: dataframe.loc[:, ['column_to_keep_one', 'column_to_keep_two']]

ahl_nhl_skaters_1719 = ahl_nhl_skaters_1719.loc[:, ['playername', 'team', 'season', 'league', 'position', 'gp', 'g', 'a', 'tp', 'ppg', 'pim', '+/-', 'link']]

#rename syntax: dataframe.rename(columns = {'old_column_name':'new_column_name'})

ahl_nhl_skaters_1719 = ahl_nhl_skaters_1719.rename(columns = {'playername':'player'})

ahl_nhl_skaters_1719
# We are now filtering out rows and columns that meet the first AND second criteria, meaning we use & instead of |. 

ahl_skaters_1718 = ahl_nhl_skaters_1719.loc[(ahl_nhl_skaters_1719.season=="2017-2018") & (ahl_nhl_skaters_1719.league=="ahl")]

nhl_skaters_1819 = ahl_nhl_skaters_1719.loc[(ahl_nhl_skaters_1719.season=="2018-2019") & (ahl_nhl_skaters_1719.league=="nhl")]

nhl_skaters_1819

# Merge syntax is first_dataframe.merge(second_dataframe, on = 'common_field', how = 'merge_type')

ahl_1718_nhl_1819 = ahl_skaters_1718.merge(nhl_skaters_1819, on = 'link', how = 'inner')

ahl_1718_nhl_1819
# Before, we dropped only one column. Now that we are dropping multiple, we must pass them as a list, which means we must enclose that list in brackets.

ahl_1718_nhl_1819 = ahl_1718_nhl_1819.drop(columns = ['season_x', 'league_x', 'season_y', 'league_y', 'player_y', 'position_y'])

ahl_1718_nhl_1819
my_columns = list(ahl_1718_nhl_1819.columns)

my_columns.remove('link')

my_columns.append('link')

my_columns
ahl_1718_nhl_1819 = ahl_1718_nhl_1819.loc[:, my_columns]
ahl_1718_nhl_1819
# Rename the columns after the inner join
ahl_1718_nhl_1819 = ahl_1718_nhl_1819.rename(columns = {'player_x':'player', 'team_x':'ahl_team', 'position_x':'position', 'gp_x':'ahl_gp', 
                                                        'g_x':'ahl_g', 'a_x':'ahl_a', 'tp_x':'ahl_p', 'ppg_x':'ahl_ppg', 'pim_x':'ahl_pim', 
                                                        '+/-_x':'ahl_+/-', 'team_y':'nhl_team', 'gp_y':'nhl_gp', 'g_y':'nhl_g', 'a_y':'nhl_a', 
                                                        'tp_y':'nhl_p', 'ppg_y':'nhl_ppg', 'pim_y':'nhl_pim', '+/-_y':'nhl_+/-'})

ahl_1718_nhl_1819

ahl_1718_nhl_1819.dtypes
# astype = "as type" (basically "cast")
# astype syntax: dataframe.series.astype(chosen_type)

ahl_1718_nhl_1819.ahl_ppg.astype(float)

# note - this is going to fail 'cause there are "-"s here
ahl_1718_nhl_1819[ahl_1718_nhl_1819.ahl_ppg=="-"]
# use numpy to return different values in the dataframe like a ternary
# re-assign to our variable
# np.where syntax: np.where(condition==True, value_to_return_if_true, value_to_return_if_false)

ahl_1718_nhl_1819.ahl_ppg = np.where(ahl_1718_nhl_1819.ahl_ppg=="-", 0, ahl_1718_nhl_1819.ahl_ppg)
ahl_1718_nhl_1819.nhl_ppg = np.where(ahl_1718_nhl_1819.nhl_ppg=="-", 0, ahl_1718_nhl_1819.nhl_ppg)

ahl_1718_nhl_1819.ahl_gp = np.where(ahl_1718_nhl_1819.ahl_gp=="-", 0, ahl_1718_nhl_1819.ahl_gp)
ahl_1718_nhl_1819.nhl_gp = np.where(ahl_1718_nhl_1819.nhl_gp=="-", 0, ahl_1718_nhl_1819.nhl_gp)

ahl_1718_nhl_1819[ahl_1718_nhl_1819.ahl_ppg=="-"]
# now we can covert to floats
ahl_1718_nhl_1819.ahl_ppg = ahl_1718_nhl_1819.ahl_ppg.astype(float) 

ahl_1718_nhl_1819.nhl_ppg = ahl_1718_nhl_1819.nhl_ppg.astype(float) 

ahl_1718_nhl_1819.ahl_gp = ahl_1718_nhl_1819.ahl_gp.astype(float) 

ahl_1718_nhl_1819.nhl_gp = ahl_1718_nhl_1819.nhl_gp.astype(float) 
# filter players that played more than 20 games in both leagues
ahl_1718_nhl_1819 = ahl_1718_nhl_1819.loc[(ahl_1718_nhl_1819.ahl_gp>=20) & (ahl_1718_nhl_1819.nhl_gp>=20)]

ahl_1718_nhl_1819
# First build two separate dataframes; one for forwards, one for defensemen.
# I guess this data doesn't actually use which defense side :p

forwards = ahl_1718_nhl_1819.loc[ahl_1718_nhl_1819.position!="D"]
defensemen = ahl_1718_nhl_1819.loc[ahl_1718_nhl_1819.position=="D"]


# Obtain two integers which denote the length of each of these dataframes, and thus the number of players who play each position within the dataframe.
# len syntax: len(object)

forward_count = len(forwards)
defenseman_count = len(defensemen)
# str syntax: str(object). 

forward_count_string = str(forward_count)
defenseman_count_string = str(defenseman_count)

# As long as you're working with multiple strings, you can print (string1 + string2). However, you will receive an error if you try to print two objects of different types.

print("We have the following number of forwards: " + forward_count_string)
print("We have the following number of defensemen: " + defenseman_count_string)<jupyter_output>We have the following number of forwards: 77
We have the following number of defensemen: 36<jupyter_text>Actual Data AnalysisOk, we've cleaned up the data and have narrowed it down to players who...:1. Played in the AHL in 2017-2018 season1. Played in the NHL in 2018-2019 season1. Played more than 20 games in both seasonsOur first goal is to:> The first thing we're going to do is determine which teams had the most NHL forwards in 2018-2019 who made the jump from the AHL the prior year.<jupyter_code># groupby syntax: dataframe.groupby('grouping_column').target_column_to_apply_function_to.function()

forward_counts = forwards.groupby('nhl_team').player.count()

forward_counts
# pandas dataframes are much easier to work with, I guess
# pd.DataFrame syntax: pd.DataFrame(object_to_convert_to_dataframe)

forward_counts = pd.DataFrame(forward_counts)
# ok, yeah, this is pretty frickin' easy :p
# sort_values syntax: dataframe.sort_values(by = 'field', ascending = True/False)

forward_counts.sort_values(by = 'player', ascending = False)
forward_counts = forward_counts.sort_values(by = 'player', ascending = False)

# this reset_index adds an "index" id column to the dataframe which is useful
forward_counts = forward_counts.reset_index()

forward_counts
# because I'm a Lightning fan, I wanna know who they had :p
tbl_forwards = forwards.loc[forwards.nhl_team == 'Tampa Bay Lightning']
tbl_forwards
# and defensemen
tbl_defensemen = defensemen.loc[defensemen.nhl_team == 'Tampa Bay Lightning']
tbl_defensemen
# let's see if we can add these together...
tbl_players = tbl_forwards.append(tbl_defensemen)
tbl_players
# nice, ok, what about players that played in Syracuse but didn't make it to the Lightning? :eyes:
all_players = forwards.append(defensemen)
crunch_players = all_players.loc[all_players.ahl_team == 'Syracuse Crunch']
crunch_players_who_did_not_play_for_lightning = crunch_players.loc[crunch_players.nhl_team != 'Tampa Bay Lightning']
crunch_players_who_did_not_play_for_lightning
# yeah, no clue who he is but, oof, that -14 for the Habs........ lol
# ok, let's get back to it
forward_counts
# set syntax: set(object)

nhl_team_set = set(nhl_skaters_1819.team)

nhl_team_set
# we're going to use `isin` so we need to convert the set to a dataframe

nhl_team_df = pd.DataFrame(nhl_team_set)

nhl_team_df
# rename the column 'cause we don't want it to be called "0", lol
nhl_team_df = nhl_team_df.rename(columns = {0:'nhl_team'})
# list syntax to convert an object to a list: list(object)

teams_in_transitioning_forwards = list(forward_counts.nhl_team)
teams_in_transitioning_forwards
#The isin function tells us whether or not these teams appear in the list.
#Isin syntax is dataframe.object.isin(values)

nhl_team_df.loc[nhl_team_df.nhl_team.isin(teams_in_transitioning_forwards)]
# ah, these ids, even tho they go to 31, there aren't actually 31 teams...
len(nhl_team_df.loc[nhl_team_df.nhl_team.isin(teams_in_transitioning_forwards)])
# the tilde returns things that fail to meet our condition
missing_teams = nhl_team_df.loc[~nhl_team_df.nhl_team.isin(teams_in_transitioning_forwards)]

missing_teams
# head syntax: dataframe.head(number_of_rows_you_want_to_see)

forward_counts.head()
# we can join the forward_counts list with the teams that are missing but they need a "player" column that defaults to 0
# assign syntax: dataframe.assign(column_name = column_values)

missing_teams = missing_teams.assign(player = 0)

missing_teams
# pd.concat syntax for concatenating two dataframes: pd.concat([first_dataframe, second_dataframe])

full_transitioning_forwards = pd.concat([forward_counts, missing_teams])

# Filter out totals since it is not actually a team.

full_transitioning_forwards = full_transitioning_forwards.loc[full_transitioning_forwards['nhl_team']!='totals']

full_transitioning_forwards
# the indices are messed up so let's reset them:

full_transitioning_forwards.reset_index().head()
# oh, lol, now there are _two_ index columns so we've gotta remove one
# Reset the index first, then drop the new 'index' column that is created by doing so.

full_transitioning_forwards = full_transitioning_forwards.reset_index().drop(columns = 'index')

# Print just the head to make sure we've done this correctly.

full_transitioning_forwards.head()
# ok cool, now we can visualize the data

# countplot syntax: sns.countplot(dataframe.column)

sns.countplot(full_transitioning_forwards.player)
# there's a... correlation coefficent matrix...?! I dunno what this is but... :p
# corrcoef syntax: np.corrcoef(array_like_object_1, array_like_object_2)

np.corrcoef(ahl_1718_nhl_1819.ahl_ppg, ahl_1718_nhl_1819.nhl_ppg)
# we really only need the second index of the first array so retrieve it :D
np.corrcoef(ahl_1718_nhl_1819.ahl_ppg, ahl_1718_nhl_1819.nhl_ppg)[0, 1]
# From the tutorial:
# > But I prefer R^2, which tells us how much of the variance in one variable can be explained by the other. To extract R^2, given correlation coefficient, simply multiply the initial value by itself. This can be done by adding **2 to the end of the number.
# /shrug but, ok, let's do it
correlation_coefficient = (np.corrcoef(ahl_1718_nhl_1819.ahl_ppg, ahl_1718_nhl_1819.nhl_ppg)[0, 1])

RSQ = correlation_coefficient**2

RSQ = round(RSQ, 2)

RSQ
# regplot syntax: sns.regplot(x = array_like_object_for_x, y = array_like_object_for_y, color = 'your_optional_choice')

sns.regplot(x = ahl_1718_nhl_1819.ahl_ppg, y = ahl_1718_nhl_1819.nhl_ppg, color = 'teal')
# there are matplot options to make the graph look better :D
sns.regplot(x = ahl_1718_nhl_1819.ahl_ppg, y = ahl_1718_nhl_1819.nhl_ppg, color = 'teal')

# plt.xlabel syntax: plt.xlabel("Label you want")

plt.xlabel("AHL Points Per Game in 2017-2018")
plt.ylabel("NHL Points Per Game in 2018-2019")
# moreeeeee
sns.regplot(x = ahl_1718_nhl_1819.ahl_ppg, y = ahl_1718_nhl_1819.nhl_ppg, color = 'teal')
plt.xlabel("AHL Points Per Game in 2017-2018")
plt.ylabel("NHL Points Per Game in 2018-2019")

# Build a single string for our R^2 value.

RSQString = "R^2 = " + str(RSQ)

# plt.text syntax: plt.text(x-location, y-location, (text))
# We want to print the R^2 value in the top left corner where there isn't much data. We know that the top left corner has an x-value of about 0.2 and y-value of about 0.7.

plt.text(0.2, 0.7, RSQString)

#We add plt.show() to the end just to show only the plot itself without the text at the top.

plt.show()
# we're gonna use this:
# > Quality of League x = (Average PPG in NHL in Year 2) / (Average PPG in league x in Year 1)
# to do some stuff
# np.mean syntax: np.mean(array_of_values)

nhl_ppg_average = np.mean(ahl_1718_nhl_1819.nhl_ppg)

ahl_ppg_average = np.mean(ahl_1718_nhl_1819.ahl_ppg)

ahl_nhl_equivalency = nhl_ppg_average/ahl_ppg_average

ahl_nhl_equivalency
# Define our NHLe function
# In line 1, we define our function as obtain_nhle_given_ahl_ppg and 1 argument to take in: ahl_ppg.
# In line 2, we calculate NHLe by multiplying AHL points per game by 82 and then multiplying that by the AHL to NHL equivalency we obtained previously.
# In line 3, we round this value to the 2nd decimal.
# In line 4, we print the NHLe.

def obtain_nhle_given_ahl_ppg(ahl_ppg):
    nhle = ahl_ppg * 82 * ahl_nhl_equivalency
    nhle = round(nhle, 2)
    print(nhle)
# If someone scores 1ppg in the AHL, we should expect x points in the NHL
obtain_nhle_given_ahl_ppg(1)
# Ok, let's return to our Lightning example now... where's that variable...
tbl_players
# ok, let's iterate and see what each player's nhle is
# ... I have no clue how python works so let's play around with this first
for _, row in tbl_players.iterrows():
    print(row)
# oh! ok this is not correct, lol, let's see...
for _, item in tbl_players.iteritems():
    print(item)
# oh, wait, actually, the row might have been right and print is just weird. let's double check...
for _, row in tbl_players.iterrows():
    print(row.player)
# ok, to actually calculate this, we need to have an arbitrary number of games so let's define another NHLe function
def obtain_nhle_given_ahl_ppg_and_nhl_gp(ahl_ppg, nhl_gp):
    nhle = ahl_ppg * nhl_gp * ahl_nhl_equivalency
    nhle = round(nhle, 2)
    print(nhle)
# alright, let's do it!
for _, row in tbl_players.iterrows():
    ahl_ppg = row.ahl_ppg
    nhl_games_played = row.nhl_gp
    nhle = obtain_nhle_given_ahl_ppg_and_nhl_gp(ahl_ppg, nhl_games_played)

    nhl_points = row.nhl_p
    player_name = row.player

    if nhle > nhl_points:
        print(player_name + ' scored less NHL points than expected')
    elif nhle == nhl_points:
        print(player_name + ' scored the expected amount of NHL points')
    else:
        print(player_name + ' scored more points than expected')
# whoops, I did this wrong, heh
def obtain_nhle_given_ahl_ppg_and_nhl_gp(ahl_ppg, nhl_gp):
    nhle = ahl_ppg * nhl_gp * ahl_nhl_equivalency
    nhle = round(nhle, 2)
    return nhle
for _, row in tbl_players.iterrows():
    ahl_ppg = row.ahl_ppg
    nhl_games_played = row.nhl_gp
    nhle = obtain_nhle_given_ahl_ppg_and_nhl_gp(ahl_ppg, nhl_games_played)

    nhl_points = float(row.nhl_p)
    player_name = row.player

    if nhle > nhl_points:
        print(player_name + ' scored less NHL points than expected')
    elif nhle == nhl_points:
        print(player_name + ' scored the expected amount of NHL points')
    else:
        print(player_name + ' scored more points than expected')<jupyter_output><NAME> scored more points than expected
<NAME> scored more points than expected
<NAME> scored more points than expected
<NAME> scored more points than expected
{'fraction_non_alphanumeric': 0.09343183511298733, 'fraction_numerical': 0.0484852247330519, 'mean_word_length': 5.16494450822809, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 22, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '628025', 'license': 'MIT', 'n_tokens_mistral': 6451, 'n_tokens_neox': 5822, 'n_words': 1621, 'path': 'TopDownHockey Tutorial.ipynb', 'repo_name': 'andrewjkerr/hockey-notebooks', 'size': '16157'}