<jupyter_start><jupyter_text>Project Purpose: In this personal project, I will create a machine learning model that will accurately predict whether the Chicago Bulls basketball team will win a given game based on that games statistics (box score). To do this, I need a dataset containing the team's stat totals for all 72 regular season basketball games in the 2020-2021 season.  After some searching, I was unable to find such a dataset. However, I was able to locate a website that contained links to all 72 box scores. From these box score pages, I will scrape the desired data. When this is complete, I will clean and model the data to fit my requirements for this project. Next, I'll use this cleaned dataset to train and test my Random Forest model.<jupyter_code>import pandas as pd
import numpy as np
import requests
from bs4 import BeautifulSoup
from sklearn.model_selection import train_test_split
from sklearn.ensemble import RandomForestClassifier
import matplotlib.pyplot as plt
from sklearn.metrics import classification_report, confusion_matrix, accuracy_score
pd.set_option('display.max_columns',None)
pd.set_option('display.max_rows', None)
url = "https://www.basketball-reference.com/teams/CHI/2021_games.html"
r = requests.get(url)
r.status_code #check to see if site allows web scraping (if 200 prints, we're good to go.)
data = r.text
soup = BeautifulSoup(data,'html.parser')
#pull all links for HTML page
my_list = []
for link in soup.find_all('a'): #Find all link tags
    my_list.append(link.get('href')) #append links
#Collect all the boxscore links
box_scores = []
for i in my_list:
    if "boxscores/2" in i:
        box_scores.append(i)
box_scores #shows path to box scores
first_half_path = "https://www.basketball-reference.com"
#Create entire link to each boxscore
box_score_urls = [] 
for i in box_scores:
    i = first_half_path+i
    box_score_urls.append(i)
box_score_urls
cols = ["Players", "MP", "FG","FGA","FG%","3PA","3P","3P%","FT","FTA","FT%","ORB","DRB",
        "TRB","AST","STL","BLK","TOV","PF","PTS","+/-","Chicago Score", "Opponent Score"]

box_score_frames_list = [] #fill up list with box score dataframes

for link in box_score_urls:
    req = requests.get(link)
    box_score_data = req.text
    box_score_soup = BeautifulSoup(box_score_data,'html.parser')
    chi_box_score_table = box_score_soup.find_all('table', id="box-CHI-game-basic") #Find all tables with an id of the box score
    table_rows= chi_box_score_table[0].find_all('tr')
    table_df = []
    
    for line in table_rows:
        scores = box_score_soup.find_all('div',{'class':'score'}) #Find Scores
        chi_score = scores[0]
        opp_score = scores[1]
        
        td = line.find_all('td')
        #site HTML has the players name under an <a tag...
        a = line.find_all('a') #Find all a tags
        a = str(a)
        a = a.replace(a[0:37],"<td class=\"right\" data-stat=\"player\">") #This will convert the a tag into looking like
                                                                            #an td tag
        a = a.replace("</a>","</td>") #Adjust the end tag to be a td end tag
        a = a.replace("]","")
        if len(td) > 0:
            a = BeautifulSoup(a,'html.parser')
            td.insert(0,a)
            td.insert(21,chi_score) #insert chicago's score as the last column of the dataframe
            td.insert(21,opp_score) #insert opp_score as the last column of the dataframe
       
        row = [line.text for line in td]
        if len(row) > 0: #Avois any rows that are empty
            table_df.append(row)
    temp_df = pd.DataFrame(table_df, columns=cols)
    box_score_frames_list.append(temp_df)
box_scores_df = pd.concat(box_score_frames_list)
box_scores_df.head(20) #This is a complete dataframe of game-by-game box scores of the Chicago Bulls 2020-2021 season
                        #still needs some cleaning though.
box_score_totals_df = box_scores_df.loc[box_scores_df['Players']==''] #Select totals rows
#Some scores are being pulled incorrectly. Next few lines fix that.
#Where ever PTS = Opponent score, we have an issue. PTS is the total points the bulls scored.
#So, if they are the same, then it would appear the scraper switched the opponent score and bulls score
box_score_totals_df["Opponent Score Fix"] = np.where(box_score_totals_df['PTS'] == box_score_totals_df["Opponent Score"], 
             box_score_totals_df["Chicago Score"],box_score_totals_df["Opponent Score"]) #Flip Chicago score and Opp score
box_score_totals_df=box_score_totals_df.iloc[1:] #First row is a duplicate of last game vs MIL
box_score_totals_df = box_score_totals_df.drop(columns=["Players","Chicago Score","Opponent Score","+/-"])
box_score_totals_df = box_score_totals_df.rename(columns={"Opponent Score Fix":"Opponent Score","PTS":"Chicago Score"})
box_score_totals_df.head()
dtype_change = box_score_totals_df[['Chicago Score','Opponent Score']].astype(int) #Change scoring columns to int types
box_score_totals_df["Formatted Chicago Score"] = dtype_change['Chicago Score']
box_score_totals_df["Formatted Opponent Score"] = dtype_change['Opponent Score']
#Create column to demonstrate if the bulls won or not.
box_score_totals_df["W/L"] = np.where(box_score_totals_df["Formatted Chicago Score"] > box_score_totals_df["Formatted Opponent Score"],"1","0")
box_score_totals_df.head()
box_score_totals_df = box_score_totals_df.drop(columns=["Chicago Score","Opponent Score"])
box_score_totals_df = box_score_totals_df.rename(columns={"Formatted Chicago Score": "Chicago Score","Formatted Opponent Score":"Opponent Score"})
box_score_totals_df.head()
box_score_totals_df["W/L"].value_counts() #Should say L: 41, W: 31
box_score_totals_df.to_csv('chicago_bulls_2019_2020.csv',index=False)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.10771620921802175, 'fraction_numerical': 0.01070257206973934, 'mean_word_length': 4.310724106324473, 'pattern_counts': {'":': 4, '<': 9, '<?xml version=': 0, '>': 11, 'https://': 2, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 3, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '366217', 'license': 'MIT', 'n_tokens_mistral': 1919, 'n_tokens_neox': 1811, 'n_words': 565, 'path': '2020_bulls_web_scraper.ipynb', 'repo_name': 'joelfeddes/chicago_bulls_games_classifcation', 'size': '5793'}