<jupyter_start><jupyter_text>Scrape Data  Import Modules<jupyter_code>import os
import requests
from bs4 import BeautifulSoup
import pandas as pd
import re
from selenium import webdriver
import time
import pickle
from pathlib import Path<jupyter_output><empty_output><jupyter_text>Scrape Game Ids<jupyter_code># def get_game_ids():
#     driver = webdriver.Chrome('/Applications/chromedriver')
#     page = 0
#     game_ids = []
#     while page < 170:
#         page += 1
#         bgg_url = 'https://boardgamegeek.com/browse/boardgame/page/' + str(page)
#         driver.get(bgg_url)
#         soup = BeautifulSoup(driver.page_source, 'html.parser')        
#         result = str(soup.find(class_='collection_table'))
#         new_game_ids = set(re.findall('boardgame?/([A-Za-z0-9-]+)', result))
#         game_ids.extend(new_game_ids)
#     return game_ids 
#     time.sleep(5)
#     filepath = 'data/raw/game_ids.pkl'
#     with open(filepath, 'wb') as pkl:
#         pickle.dump(game_ids, pkl)

#     driver.close()
    
# game_ids = get_game_ids()

# filepath = '/Users/tim/src/Metis/project_2/data/raw/game_ids.pkl'
# with open(filepath, 'wb') as pkl:
#     pickle.dump(game_ids, pkl)<jupyter_output><empty_output><jupyter_text>Scrape Game Data<jupyter_code>driver = webdriver.Chrome('/Applications/chromedriver')
game_id = 1
game_url = 'https://boardgamegeek.com/boardgame/' + str(game_id)
driver.get(game_url)
soup = BeautifulSoup(driver.page_source, 'html.parser')
# def pickle_game_data(source, game_id):
#     filepath = 'data/raw/second_scrape/' + 'id' + str(game_id) + '.pkl'
#     with open(filepath, 'wb') as pkl:
#         pickle.dump(source, pkl)

# def scrape_game_data(game_id, driver):
#     game_url = 'https://boardgamegeek.com/boardgame/' + str(game_id)
#     driver.get(game_url)
#     pickle_game_data(driver.page_source, game_id)
#     return BeautifulSoup(driver.page_source, 'html.parser')
    
# def scrape_all_games_data(game_ids):
#     game_soups = []
#     for num in game_ids:
#         if num in num_list:
#             driver = webdriver.Chrome('/Applications/chromedriver')
#             soup2 = scrape_game_data(num, driver)
#         #pickle_game_data(soup2, num)
#             game_soups.append(soup2)
#             time.sleep(5)
#             driver.close()
#     return game_soups
    
# game_soups = scrape_all_games_data(game_ids)
game_info_data = []
filepath = '/Users/tim/src/Metis/project_2/data/raw/first_scrape/'

for file in os.listdir(filepath):
    if Path(filepath + file).suffix == '.pkl':
        with open(filepath + file, 'rb') as pkl:
            game_string = pickle.load(pkl)
        game_info_data.append(game_string)
    else:
        pass<jupyter_output><empty_output><jupyter_text>Helper Functions to Collect Game Data from Soups<jupyter_code># def get_title(game):
#     title_string = game.find('title').text
#     title = title_string.split('|')[0].strip() 
#     return title

# def get_rating(game):
#     rating_regex = re.compile('has-rating-[0-9]')
#     rating_string = game.find('a', {'class': rating_regex}).text
#     rating = float(re.findall('\d+\.\d',rating_string)[0])
#     rating = float(re.findall('\d+\.\d',rating_string)[0])
#     return rating

# def get_release_year(game): 
#     release_year_string = game.find('span', class_='game-year').text
#     release_year = int(re.findall('\d+',release_year_string)[0])
#     return release_year

# def get_players_string(game):
#     players_string = game.find('div', class_='gameplay-item-primary').text
#     return players_string
    
# def get_player_min(game):
#     players_string = get_players_string(game)
#     player_min = int(re.findall('\d+',players_string)[0])
#     return player_min

# def get_player_max(game):
#     players_string = get_players_string(game)
#     try:
#         player_max = int(re.findall('\d+',players_string)[1].strip())
#     except:
#         player_max = player_min
#     return player_max

# def get_playtime_string(game):
#     playtime_string = game.find('li', class_='gameplay-item').findNextSibling().text
#     return playtime_string

# def get_playtime_min(game):
#     playtime_string = get_playtime_string(game)
#     playtime_min = int(re.findall('\d+',playtime_string)[0].strip())
#     return playtime_min

# def get_playtime_max(game):
#     playtime_string = get_playtime_string(game)
#     try:
#         playtime_max = int(re.findall('\d+',playtime_string)[1].strip())
#     except:
#         playtime_max = playtime_min
#     return playtime_max

# def get_complexity(game):
#     complexity_regex = re.compile('gameplay-weight-')
#     complexity_string_el = game.find('span', {'class': complexity_regex})
#     if complexity_string_el == None:
#         return "Stop"
#     complexity_string = complexity_string_el.text
#     complexity = float(complexity_string.strip())
#     return complexity

# def get_genre(game):
#     genre_string = game.find('div', class_='feature-description').text
#     genre = genre_string.strip().split(',')
#     return genre

# def get_list_price(game):
#     list_price_string = game.find('div', class_='summary-sale-item-price summary-sale-item-price-retail').text
#     list_price_clean = list_price_string.replace('$', '').strip()
#     if len(list_price_clean) == 0:
#         return None
#     list_price = float(list_price_clean)
#     return list_price

# def get_expansions(game):
#     expansions_string = game.find('span', class_='text-muted pull-right').text
#     expansions = int(expansions_string.strip())
#     if expansions == None:
#         return int(0)
#     return expansions

# def get_awards(game):
#     awards_string_el = game.find('div', class_='panel panel-top game-awards panel-condensed')
#     if awards_string_el == None:
#         return None
#     awards_string = awards_string_el.text
#     awards = awards_string[30:-20].strip().split('  ')
#     awards_num = len(awards)
#     return awards_num

# def print_game_info(games_data):
#     for game in games_data:
#         print(str(get_title(game)))
#         print(str(get_rating(game)))
#         print('Release Year:' + str(get_release_year(game)))
#         print('Player Min:' + str(get_player_min(game)))
#         print('Player Max:' + str(get_player_max(game)))
#         print('Playtime Min:' + str(get_playtime_min(game)))
#         print('Playtime Max:' + str(get_playtime_max(game)))
#         print('Genre:' + str(get_genre(game)))
#         print('Complexity:' + str(get_complexity(game)))
#         print('List Price:' + str(list_price(game)))
#         print('Number of Expansions:' + str(get_expansions(game)))
#         print('Number of Awards:' + str(get_awards(game)))
        
# def get_game_info(game):
#     game_info = {
#         "title": get_title(game),
#         "rating": get_rating(game),
#         "release_year": get_release_year(game),
#         "player_min": get_player_min(game),
#         "player_max": get_player_max(game),
#         "playtime_min": get_playtime_min(game),
#         "playtime_max": get_playtime_max(game),
#         "genre": get_genre(game),
#         "complexity": get_complexity(game),
#         "list_price": get_list_price(game),
#         "expansions": get_expansions(game),
#         "awards": get_awards(game)
#      }
#     return game_info

# def get_all_games_info(games_data):
#     all_games_dict = {}
#     for html_source in games_data:
#         game = BeautifulSoup(html_source, 'html.parser')
#         title = get_title(game)
#         all_games_dict[title] = get_game_info(game)
#     return all_games_dict
# filepath = '/Users/tim/src/Metis/project_2/data/interim/all_games.pkl'
# with open(filepath, 'wb') as pkl:
#     pickle.dump(all_games.pkl)<jupyter_output><empty_output><jupyter_text>Collect Game Data and Convert to DataFrame<jupyter_code>df = pd.DataFrame(all_games).T
df
# filepath = '/Users/tim/src/Metis/project_2/data/interim/all_games_df.pkl'
# with open(filepath, 'wb') as pkl:
#     pickle.dump(game_ids, pkl)<jupyter_output><empty_output><jupyter_text>Clean Data  Import Modules<jupyter_code>import patsy
import seaborn as sns
import matplotlib.pyplot as plt

import statsmodels.api as sm
import statsmodels.formula.api as smf
from sklearn.linear_model import LinearRegression
from sklearn.model_selection import train_test_split

%matplotlib inline<jupyter_output><empty_output><jupyter_text>Load DataFrame<jupyter_code>filepath = '/Users/tim/src/Metis/project_2/data/interim/all_games_df.pkl'
with open(filepath, 'rb') as pkl:
    games_df = pickle.load(pkl)<jupyter_output><empty_output><jupyter_text>Inspect Data<jupyter_code>games_df.info()<jupyter_output><class 'pandas.core.frame.DataFrame'>
Index: 8590 entries, Dead Last to My First Rummikub
Data columns (total 11 columns):
awards          8590 non-null object
complexity      8587 non-null object
expansions      8586 non-null object
genre           8586 non-null object
list_price      2078 non-null object
player_max      8569 non-null object
player_min      8569 non-null object
playtime_max    8381 non-null object
playtime_min    8381 non-null object
rating          8587 non-null object
release_year    8503 non-null object
dtypes: object(11)
memory usage: 805.3+ KB<jupyter_text>Convert Numerical Values to Floats and Remove Remaining Column<jupyter_code>games_df_nums = games_df.infer_objects()
games_df_nums = games_df_nums.select_dtypes(exclude=['object']).copy()
games_df_nums.info()<jupyter_output><class 'pandas.core.frame.DataFrame'>
Index: 8590 entries, Dead Last to My First Rummikub
Data columns (total 10 columns):
awards          8590 non-null float64
complexity      8587 non-null float64
expansions      8586 non-null float64
list_price      2078 non-null float64
player_max      8569 non-null float64
player_min      8569 non-null float64
playtime_max    8381 non-null float64
playtime_min    8381 non-null float64
rating          8587 non-null float64
release_year    8503 non-null float64
dtypes: float64(10)
memory usage: 738.2+ KB<jupyter_text>Remove Null Values<jupyter_code>games_df2 = games_df_nums[games_df_nums.rating.notnull()]
games_df2.drop(['list_price'], axis=1)
games_df2.dropna()<jupyter_output><empty_output><jupyter_text>Reinspect DataFrame<jupyter_code>games_df2.info()
games_df2.describe()<jupyter_output><empty_output><jupyter_text>Check for Multicollinearity<jupyter_code>plt.figure(figsize=(12,8))
sns.heatmap(games_df2.corr(), cmap="seismic", vmin=-1, vmax=1);<jupyter_output><empty_output><jupyter_text>We can see that playtime_max and playtime_min display colinearity, meaning that I will need to drop one in my model. However, neither variable is strongly correlated with ratings, my target variable, so both columns will likely be dropped.  Though complexity demonstrates a strongest correlation among the feature variables, it is a modest correlation.<jupyter_code>games_df2.loc[:,'good'] = 1
games_df2.loc[games_df2.rating <= 6, 'good'] = 0
sns.pairplot(games_df4, hue='good')<jupyter_output><empty_output><jupyter_text>Though the scaling of the variables makes it difficult to get a clear visualization of the data, we can still gain some interesting insights. For one, there are quite a few outliers scattered throughout the dataset. These will need to be accounted for later.   MVP: First Model  Modeling with Statsmodels<jupyter_code># Create feature matrix (X) and target vector (y)
y, X = patsy.dmatrices('rating ~ awards + complexity + expansions + player_max + player_min + playtime_max + playtime_min + release_year', data=games_df2, return_type="dataframe")

# Split the data into training and test sets
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=.2, random_state=13)

# Create model
model = sm.OLS(y_train, X_train)

# Fit your model to your training set
fit1 = model.fit()

fit1.summary()<jupyter_output><empty_output><jupyter_text>An r-squared of 0.412 is not a bad place to start. Hopefully with a little bit of cleaning and feature engineering, we can improve that score.As I expected, I will have to remove player_max given its exorbitant p-value. It also looks like playtime_max will have to go as well. I also see that the condition number is quite large, confirming the multicollinearity issues that I spotted in my heatmap earlier.<jupyter_code>plt.figure(figsize=(12,8))
plt.scatter(fit1.predict(), fit1.resid,alpha=0.1);<jupyter_output><empty_output><jupyter_text>Collect More Data  Helper Functions to Collect Number of Ratings from Soups<jupyter_code># def get_title(game):
#     title_string = game.find('title').text
#     title = title_string.split('|')[0].strip() 
#     return title

# def get_rating_num(game):
#     rating_num_string = game.find_all('div', class_='game-header-title-info')[1].text
#     if rating_num_string == None:
#         return None
    
#     rating_num = re.findall('\)(.*)(?=Ratings)',rating_num_string)[0].strip()

#     if "K" in rating_num:
#         if "." in rating_num:
#             rating_num = rating_num.replace('K', '00').replace('.', '')
#         else:
#             rating_num = rating_num.replace('K', '000')
#     return rating_num
        
# def get_game_info(game):
#     game_info = {"rating_num": get_rating_num(game)}
#     return game_info

# def get_all_games_info(games_data):
#     all_games_dict = {}
#     for index, html_source in enumerate(games_data):
#         game = BeautifulSoup(html_source, 'html.parser')
#         title = get_title(game)
#         try:
#             all_games_dict[title] = get_game_info(game)
#         except IndexError:
#             print(f"Error with {title}: regex not matched")
#         if index % 100 == 0:
#             print(f'Have scanned {index+1} out of {len(games_data)} records')
#     return all_games_dict<jupyter_output><empty_output><jupyter_text>Collect Number of Ratings and Convert to Dataframe<jupyter_code># games_rating_nums = get_all_games_info(game_info_data)
# rating_num_df = pd.DataFrame(games_rating_nums).T
# rating_num_df
# filepath ='/Users/tim/src/Metis/project_2/data/interim/rating_num.pkl'
# with open(filepath, 'wb') as pkl:
#     pickle.dump(games_rating_nums, pkl)

# filepath = '/Users/tim/src/Metis/project_2/data/interim/rating_num_df.pkl'
# with open(filepath, 'wb') as pkl:
#     pickle.dump(rating_num_df, pkl)<jupyter_output><empty_output><jupyter_text>Clean Dataframe Again  Load in Dataframes<jupyter_code>filepath = '/Users/tim/src/Metis/project_2/data/interim/all_games_df.pkl'
with open(filepath, 'rb') as pkl:
    all_games_df = pickle.load(pkl)

filepath = '/Users/tim/src/Metis/project_2/data/interim/rating_num_df.pkl'
with open(filepath, 'rb') as pkl:
    rating_num_df = pickle.load(pkl)<jupyter_output><empty_output><jupyter_text>Join Dataframes<jupyter_code>df = all_games_df.join(rating_num_df)<jupyter_output><empty_output><jupyter_text>Inspect Data<jupyter_code>df.info()<jupyter_output><class 'pandas.core.frame.DataFrame'>
Index: 8590 entries, Dead Last to My First Rummikub
Data columns (total 12 columns):
awards          8590 non-null object
complexity      8587 non-null object
expansions      8586 non-null object
genre           8586 non-null object
list_price      2078 non-null object
player_max      8569 non-null object
player_min      8569 non-null object
playtime_max    8381 non-null object
playtime_min    8381 non-null object
rating          8587 non-null object
release_year    8503 non-null object
rating_num      8505 non-null object
dtypes: object(12)
memory usage: 1.2+ MB<jupyter_text>Clean rating_num Column<jupyter_code>df['rating_num'] = df['rating_num'].str.extract('(\d+)$', expand=True)<jupyter_output><empty_output><jupyter_text>Convert Numerical Variables to Floats<jupyter_code>df = df.infer_objects()
df['rating_num'] = df['rating_num'].apply(pd.to_numeric)
df.info()
df.describe()<jupyter_output><empty_output><jupyter_text>Adjust Outliers<jupyter_code>df.loc[df.player_max > 12, 'player_max'] = 12

df.loc[df.release_year < 1950, 'release_year'] = 1950
df.loc[df.release_year > 2019, 'release_year'] = 1950
df['years_released'] = 2019 - df['release_year']<jupyter_output><empty_output><jupyter_text>Filter Out Nulls and Unneeded Data<jupyter_code>df_filtered = df[df.complexity > 0].drop(index = 'Scene It? Seinfeld').drop(['expansions', 'list_price'], axis = 1)
df_filtered = df_filtered[df.rating.notnull()].dropna()
df_filtered = df_filtered[df.rating_num >= 100]

df_filtered2 = df[df.complexity > 0].drop(index = 'Scene It? Seinfeld').drop(['expansions'], axis = 1)
df_filtered2 = df_filtered2[df.rating.notnull()].dropna()
df_filtered2 = df_filtered2[df.rating_num >= 100]
df_filtered.info()
df_filtered2.info()<jupyter_output><class 'pandas.core.frame.DataFrame'>
Index: 1989 entries, Dead Last to Architects of the West Kingdom
Data columns (total 12 columns):
awards            1989 non-null float64
complexity        1989 non-null float64
genre             1989 non-null object
list_price        1989 non-null float64
player_max        1989 non-null float64
player_min        1989 non-null float64
playtime_max      1989 non-null float64
playtime_min      1989 non-null float64
rating            1989 non-null float64
release_year      1989 non-null float64
rating_num        1989 non-null float64
years_released    1989 non-null float64
dtypes: float64(11), object(1)
memory usage: 202.0+ KB<jupyter_text>Feature Engineering  Import Modules<jupyter_code>import patsy
import pickle
import pandas as pd

import seaborn as sns
import matplotlib.pyplot as plt

from sklearn.linear_model import LinearRegression
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import StandardScaler, PolynomialFeatures
from sklearn.model_selection import KFold, cross_val_score

%matplotlib inline<jupyter_output><empty_output><jupyter_text>Load DataFrames<jupyter_code>filepath = '/Users/tim/src/Metis/project_2/data/interim/clean_df.pkl'
with open(filepath, 'rb') as pkl:
    df = pickle.load(pkl)

filepath = '/Users/tim/src/Metis/project_2/data/interim/clean_df_list_price.pkl'
with open(filepath, 'rb') as pkl:
    df2 = pickle.load(pkl)<jupyter_output><empty_output><jupyter_text>Get Dummy Variables for Genres<jupyter_code>genres_df = pd.get_dummies(df_filtered['genre'].apply(pd.Series).stack()).sum(level=0)
genres_df

genres_df2 = pd.get_dummies(df_filtered2['genre'].apply(pd.Series).stack()).sum(level=0)
genres_df2<jupyter_output><empty_output><jupyter_text>Merge Dataframes<jupyter_code>clean_df = df_filtered.join(genres_df)
clean_df

clean_df2 = df_filtered2.join(genres_df2)
clean_df2
# filepath = '/Users/tim/src/Metis/project_2/data/interim/clean_df.pkl'
# with open(filepath, 'wb') as pkl:
#     pickle.dump(clean_df, pkl)

# filepath = '/Users/tim/src/Metis/project_2/data/interim/clean_df_list_price.pkl'
# with open(filepath, 'wb') as pkl:
#     pickle.dump(clean_df2, pkl)<jupyter_output><empty_output><jupyter_text>Clean Duplicate Columns<jupyter_code>df.rename(index=str, columns={"Children's":"childrens1",
                              "\xa0  Children's":"childrens2",
                              "Uncategorized":"uncategorized",
                              "Abstract":"abstract",
                              "\xa0  Wargames":"wargames2",
                              "\xa0  Customizable":"customizable2",
                              "\xa0  Family":"family2",
                              "\xa0  Party":"party2",
                              "\xa0  Thematic":"thematic2",
                              "\xa0  Strategy":"strategy2"}, inplace=True)

df2.rename(columns={"Children's":"childrens1",
                              "\xa0  Children's":"childrens2",
                              "Uncategorized":"uncategorized",
                              "Abstract":"abstract",
                              "\xa0  Wargames":"wargames2",
                              "\xa0  Family":"family2",
                              "\xa0  Party":"party2",
                              "\xa0  Thematic":"thematic2",
                              "\xa0  Strategy":"strategy2"}, inplace=True)<jupyter_output><empty_output><jupyter_text>Combine Duplicate Columns<jupyter_code>df['children'] = df['childrens1']+df['childrens2']
df['wargames'] = df['Wargames']+df['wargames2']
df['customizable'] = df['Customizable']+df['customizable2']
df['family'] = df['Family']+df['family2']
df['party'] = df['Party']+df['party2']
df['thematic'] = df['Thematic']+df['thematic2']
df['strategy'] = df['Strategy']+df['strategy2']

df2['children'] = df2['childrens1']+df2['childrens2']
df2['wargames'] = df2['Wargames']+df2['wargames2']
df2['family'] = df2['Family']+df2['family2']
df2['party'] = df2['Party']+df2['party2']
df2['thematic'] = df2['Thematic']+df2['thematic2']
df2['strategy'] = df2['Strategy']+df2['strategy2']<jupyter_output><empty_output><jupyter_text>Delete Duplicate Columns<jupyter_code>df.drop(columns=['childrens1', 'childrens2', 'Wargames', 'wargames2', 'Customizable', 'customizable2',
                 'Family', 'family2', 'Party', 'party2', 'Thematic', 'thematic2', 'Strategy', 'strategy2'], inplace=True)

df2.drop(columns=['childrens1', 'childrens2', 'Wargames', 'wargames2', 'Customizable', 'Family', 'family2', 'Party', 'party2', 'Thematic', 'thematic2', 'Strategy', 'strategy2'], inplace=True)<jupyter_output><empty_output><jupyter_text>Create Log of Playtime Variables<jupyter_code>df['playtime_min_log'] = np.log(df['playtime_min'])
df['playtime_max_log'] = np.log(df['playtime_max'])<jupyter_output><empty_output><jupyter_text>Create Polynomials<jupyter_code>df['complexity2'] = df['complexity'] ** 2
df['years_released2'] = df['years_released'] ** 2<jupyter_output><empty_output><jupyter_text>Create Interations<jupyter_code>df['playtime_log_diff'] = df['playtime_max_log'] - df['playtime_min_log']
#df2['playtime_log_diff'] = df2['playtime_max_log'] - df2['playtime_min_log']
#df['awards_per_year'] = df['awards']/ df['years_released']
#df2['awards_per_year'] = df2['awards']/ df2['years_released']


#df['playtime_diff'] = df['playtime_max'] - df['playtime_min']
#df['complex_player_max'] = df['complexity']/ df['player_max']
#df['complex2_player_max'] = df['complexity2']/ df['player_max']
#df['complex2_player_min'] = df['complexity2']/ df['player_min']
#df['complex_player_min'] = df['complexity']/ df['player_min']
#df['playtime_min_log_strategy'] = df['strategy']/ df['playtime_min_log']
#df['playtime_max_log_strategy'] = df['strategy']/ df['playtime_max_log']
#df['playtime_max_strategy'] = df['strategy']/ df['playtime_max']
#df['playtime_max_strategy'] = df['strategy']/ df['playtime_max']
#df['complex_playtime_max_log'] = df['playtime_max_log'] * df['complexity']
#df['complex_playtime_min_log'] = df['playtime_min_log'] * df['complexity']<jupyter_output><empty_output><jupyter_text>Re-Model  Separate Features from Target and Scale<jupyter_code># Create feature matrix (X) and target vector (y)
y, X = patsy.dmatrices('rating ~ awards + complexity + playtime_log_diff + player_min + years_released + children + strategy + thematic + uncategorized + wargames', data=df, return_type="dataframe")
#y2, X2 = patsy.dmatrices('rating ~ awards + complexity + playtime_log_diff + player_max + player_min + years_released + list_price + children + strategy + thematic + uncategorized + wargames', data=df2, return_type="dataframe")

X, y = np.array(X), np.array(y)
#X2, y2 = np.array(X2), np.array(y2)
X, X_test, y, y_test = train_test_split(X, y, test_size=.2, random_state=13)
#X2, X2_test, y2, y2_test = train_test_split(X2, y2, test_size=.2, random_state=13)

scaler = StandardScaler()

X_train_scaled = scaler.fit_transform(X)
#X2_train_scaled = scaler.fit_transform(X2)

X_test_scaled = scaler.transform(X_test)<jupyter_output><empty_output><jupyter_text>Cross-Validate<jupyter_code>lr = LinearRegression()

kf = KFold(n_splits=5, shuffle=True, random_state = 22)
cross_val_score(lr, X_train_scaled, y, cv=kf, scoring='r2')
np.mean(cross_val_score(lr, X_train_scaled, y, cv=kf, scoring='r2'))<jupyter_output><empty_output><jupyter_text>Predict<jupyter_code>lr = LinearRegression()
fit2 = lr.fit(X_train_scaled,y)
y_pred = lr.predict(X_test_scaled)
lr.score(X_test_scaled,y_test)
y_pred_train = lr.predict(X_train_scaled)
MYX=np.arange(4,10, 5)
MYY=-MYX + 5.9

err = y - y_pred_train
plt.scatter(y_pred_train, err)
plt.plot()
plt.plot(MYX, MYY, 'r')

mask_below = (err < (-y_pred_train + 5.9))<jupyter_output><empty_output><jupyter_text>Trying to Figure Out This Trend<jupyter_code># X_train_scaled[:5,:]
# X_train_df = pd.DataFrame(X_train_scaled, columns=['intercept', 'awards', 'complexity', 'playtime_log_diff', 'player_min', 'years_released', 'children', 'strategy', 'thematic', 'uncategorized',  'wargames'])
# X_train_df.loc[mask_below[:,0],:]

# features = ['awards', 'complexity', 'playtime_log_diff', 'player_min', 'years_released', 'children', 'strategy', 'thematic', 'uncategorized',  'wargames']
# features_df = df[features]
# features_df["intercept"] = 0
# scaled_data = scaler.transform(features_df)
# scaled_df = pd.DataFrame(scaled_data, index=df.index, columns=[1].extend(features))

# scaled_df.columns = [1].extend(features)
# scaled_df.head()
# low_errors = scaled_df.merge(X_train_df.loc[mask_below[:,0],:])
# low_errors.head()<jupyter_output><empty_output><jupyter_text>Plots<jupyter_code>features = ['awards', 'complexity', 'playtime_log_diff', 'player_min', 'years_released', 'children', 'strategy', 'thematic', 'uncategorized', 'wargames']
list(zip(features, fit2.coef_.reshape(-1)))
coef_df = pd.DataFrame(zip(features, fit2.coef_.reshape(-1)), )
coef_df.columns = ['Features', 'Coefficients']
sorted_df = coef_df.sort_values('Coefficients').reset_index(drop=True)
plt.figure(figsize=(12,8))
sns.barplot(x=sorted_df['Features'], y=sorted_df['Coefficients'], orient = 'v', color = '#f57133', errcolor = 'red')
sns.set(font_scale=2)
plt.xticks(rotation = 45, horizontalalignment="right")
#plt.xlabel('Prediction', size = 20, weight='bold')
#plt.ylabel('Error', size=20,  weight='bold')
pred_df = pd.DataFrame({'Actual': y_test.reshape(-1), 
                        'Predicted': y_pred.reshape(-1)})  
#pred_df
err = y_pred - y_test
plt.figure(figsize=(12,8))
final_resid = sns.residplot(x=y_pred, y=err, scatter_kws={"s": 80, 'alpha':0.3})

MYX=np.arange(4,10, 5)
MYY=MYX - 5.9
sns.set(font_scale=2)
plt.xlabel('Prediction', size = 20, weight='bold')
plt.ylabel('Error', size=20,  weight='bold')
plt.plot(MYX, MYY)
final_resid_plot = final_resid.get_figure()  
final_resid_plot.savefig("final_resid_plot.png", format='png', bbox_inches='tight');
err
mask_below = (err < (y_pred - 5.9))
plt.figure(figsize=(12,8))
plt.scatter(x=y_pred[mask_below], y=err[mask_below])
plt.scatter(x=y_pred[~mask_below], y=err[~mask_below])

MYX=np.arange(4,10, 5)
MYY=MYX - 5.9
sns.set(font_scale=2)
plt.xlabel('Prediction', size = 20, weight='bold')
plt.ylabel('Error', size=20,  weight='bold')
plt.plot(MYX, MYY)
y_test.shape<jupyter_output><empty_output><jupyter_text>Checking for Multicollinearity<jupyter_code>plt.figure(figsize=(25,25))
sns.heatmap(df.corr(), cmap="seismic", annot=True, vmin=-1, vmax=1);
plt.figure(figsize=(25,25))
sns.pairplot(df, height=3, aspect=3);<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13988324705364028, 'fraction_numerical': 0.022616294011822153, 'mean_word_length': 4.679316096747289, 'pattern_counts': {'":': 33, '<': 166, '<?xml version=': 0, '>': 167, 'https://': 3, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '40', 'dir': 'jupyter-structured-clean-dedup', 'id': '521523', 'license': 'MIT', 'n_tokens_mistral': 10177, 'n_tokens_neox': 9706, 'n_words': 1736, 'path': 'Notebooks/compiled_notebooks.ipynb', 'repo_name': 'broco347/tabletop-success', 'size': '27237'}