<jupyter_start><jupyter_text>Training Split and EDA<jupyter_code>from sklearn.model_selection import train_test_split
import matplotlib as plt
import matplotlib.ticker as ticker
import matplotlib.pylab as pylab
import seaborn as sns
params = {'legend.fontsize': 'x-large',
          'figure.figsize': (15, 10),
         'axes.labelsize': 'x-large',
         'axes.titlesize':'x-large',
         'xtick.labelsize':'x-large',
         'ytick.labelsize':'x-large'}
pylab.rcParams.update(params)

sns.set(font_scale=2.5)

formatter = ticker.FormatStrFormatter('$%1.2f') # to format Y-axis for Price
# X, y = cars.drop('price',axis=1), cars['price']
x_cat_vars = ['Card Material','Rarity','Category','Element','set',]
x_flag_vars = ['Has EX Burst', 'Full Art',]
x_num_vars = ['Desc Length','log(Cost)','Ability Count','Power',]
x_vars = x_cat_vars + x_flag_vars + x_num_vars
X, y = all_cards_reg[x_vars], all_cards_reg['TCG Market Price']

# hold out 20% of the total data for final testing
X_raw, X_test_raw, y_, y_test = train_test_split(X, y, test_size=.2, random_state=10)

df_raw = X_raw.join(y_)

len(df_raw)
df_raw.head()
ax0 = sns.histplot(data=df_raw,x='TCG Market Price',bins=100)
ax0.set(xlim=(-1,20))
ax0.xaxis.set_major_formatter(formatter)

plt.pyplot.savefig('price histogram.svg')
sns.set(font_scale=1)

sns.pairplot(df_raw, plot_kws=dict(alpha=.1, edgecolor='none'));
sns.set(font_scale=2.5)

ax1 = sns.boxplot(x='Card Material',y='TCG Market Price',data=df_raw);
ax1.set(ylim=(-1,10))
ax1.yaxis.set_major_formatter(formatter)
ax2 = sns.boxplot(x='Full Art',y='TCG Market Price',data=df_raw)
ax2.yaxis.set_major_formatter(formatter)
ax2.set(ylim=(-1,40))
fig, axs0 = plt.pyplot.subplots(ncols=2,sharey=True)
sns.boxplot(x='Full Art',y='TCG Market Price',data=df_raw,ax=axs0[1])
sns.boxplot(x='Card Material',y='TCG Market Price',data=df_raw,ax=axs0[0], order=['Normal','Foil'])
axs0[0].yaxis.set_major_formatter(formatter)
y_ax_label_1 = axs0[1].axes.get_yaxis().get_label()
y_ax_label_1.set_visible(False)
axs0[0].set(ylim=(-1,45))
plt.pyplot.savefig('art boxplot.svg')
ax3 = sns.boxplot(x='Rarity',y='TCG Market Price',data=df_raw,order=['Common','Rare','Hero','Starter','Promo','Legend']);

ax3.yaxis.set_major_formatter(formatter)
ax3.set(ylim=(-1,45));
plt.pyplot.savefig('rarity boxplot.svg')
ax4 = sns.boxplot(x='Has EX Burst',y='TCG Market Price',data=df_raw);
ax4.set(ylim=(0,4))
sns.set(font_scale=1)

ax5 = sns.boxplot(x='Element',y='TCG Market Price',data=df_raw);
ax5.set(ylim=(0,15))
sns.set(font_scale=2.5)

fig, axs1 = plt.pyplot.subplots(ncols=4,sharey=True)
sns.regplot(x='Desc Length', y='TCG Market Price', line_kws={"color": "red"}, data=df_raw, ax=axs1[0])
sns.regplot(x='log(Cost)', y='TCG Market Price', line_kws={"color": "red"}, data=df_raw, ax=axs1[1])
sns.regplot(x='Ability Count', y='TCG Market Price', line_kws={"color": "red"}, data=df_raw, ax=axs1[2])
sns.regplot(x='Power', y='TCG Market Price', line_kws={"color": "red"}, data=df_raw, ax=axs1[3])

axs1[0].yaxis.set_major_formatter(formatter)
axs1[0].set(ylim=(-1,45))
axs1[3].set(xlim=(-500,10500))

for i in range(1,4):
    axs1[i].axes.get_yaxis().get_label().set_visible(False)
    
plt.pyplot.savefig('card attributes regplot.svg')<jupyter_output><empty_output><jupyter_text>Validation Preprocessing of Dummy Vars<jupyter_code>def create_X_vars(df):
    """
    takes in a training/validation/test dataset for the regression
    and returns a dataframe that joins the numerical variables with
    the dummy variables converted from categorical variables
    """
    x_cat_vars = ['Card Material','Rarity','set',] # Removed Element, Category
    x_flag_vars = ['Full Art', "Has EX Burst",] 
    x_num_vars = ['log(Cost)','Ability Count','Power','Desc Length',]

    X_ = df[x_flag_vars+x_num_vars]
#     X_ = df[x_flag_vars]
#     for col in x_num_vars:
#         if col in ['Cost']:
#             X_.join(np.log(df[col]))
#         else:
#             X_.join(df[col])
    for col in x_cat_vars:
        if col == 'Element':
            cleaned = df['Element'].str.split(';',expand=True).stack()
            dummies = pd.get_dummies(cleaned).groupby(level=0).sum()
        else:
            dummies = pd.get_dummies(df[col],drop_first=True)

        X_ = X_.join(dummies)

    return X_
X_=create_X_vars(X_raw)
X_train, X_val, y_train, y_val = train_test_split(X_, y_, test_size=.25, random_state=3)
X_test = create_X_vars(X_test_raw)
X_train.shape, y_train.shape, X_val.shape,y_val.shape, X_test.shape, y_test.shape<jupyter_output><empty_output><jupyter_text>Validate<jupyter_code># Import libraries for regression modelling
from sklearn.linear_model import LinearRegression
from sklearn.preprocessing import PolynomialFeatures
from sklearn.preprocessing import StandardScaler
from sklearn.linear_model import LassoCV, RidgeCV<jupyter_output><empty_output><jupyter_text>Feature Transformations<jupyter_code>poly = PolynomialFeatures(degree=2,interaction_only=True,include_bias = False)
s = StandardScaler(with_mean=False)

X_train_poly = poly.fit_transform(X_train)
X_val_poly = poly.fit_transform(X_val)
X_test_poly = poly.fit_transform(X_test)

X_train_poly_scaled = s.fit_transform(X_train_poly)
X_val_poly_scaled = s.fit_transform(X_val_poly)
X_test_poly_scaled = s.fit_transform(X_test_poly)
print(X_train_poly_scaled.shape)
print(X_train_poly.shape)
print(y_train.shape)<jupyter_output>(2235, 325)
(2235, 325)
(2235,)<jupyter_text>Linear Regression: Normal Features<jupyter_code>reg = LinearRegression()
reg.fit(X_train, y_train)

print(f'Linear Regression on Training Data R^2: {reg.score(X_train, y_train)}')
print(f'Linear Regression on Validation Data R^2: {reg.score(X_val, y_val)}')<jupyter_output>Linear Regression on Training Data R^2: 0.41575545253597246
Linear Regression on Validation Data R^2: 0.4204576153833922<jupyter_text>Linear Regression: Polynomial Features<jupyter_code>poly_reg = LinearRegression()
poly_reg.fit(X_train_poly, y_train)

print(f'Poly Regression on Training Data R^2: {poly_reg.score(X_train_poly, y_train)}')
print(f'Poly Regression on Validation Data R^2: {poly_reg.score(X_val_poly, y_val)}')<jupyter_output>Poly Regression on Training Data R^2: 0.6247834098821026
Poly Regression on Validation Data R^2: 0.5049139912265036<jupyter_text>Lasso Regression<jupyter_code>m_lasso = LassoCV()
m_lasso.fit(X_train_poly_scaled,y_train)

print(f'LASSO Regression on Training Data R^2: { m_lasso.score(X_train_poly_scaled,y_train)}')
print(f'LASSO Regression on Validation Data R^2: { m_lasso.score(X_val_poly_scaled,y_val)}')<jupyter_output>C:\Users\wlee\AppData\Local\Continuum\anaconda3\lib\site-packages\sklearn\model_selection\_split.py:1978: FutureWarning: The default value of cv will change from 3 to 5 in version 0.22. Specify it explicitly to silence this warning.
  warnings.warn(CV_WARNING, FutureWarning)<jupyter_text>Ridge Regression<jupyter_code># Identify optimal alpha for Ridge Regression 

my_a= []
my_r2 = []
for a in np.linspace(710,711,11):
    m_ridge = RidgeCV(alphas=[a])
    m_ridge.fit(X_train_poly_scaled,y_train)
    my_a.append(m_ridge.alpha_)
    my_r2.append(m_ridge.score(X_val_poly_scaled,y_val))

max_index = my_r2.index(max(my_r2))
my_a[max_index]

print(my_a[max_index])
plt.pyplot.plot(my_a,my_r2)
m_ridge = RidgeCV(alphas=[710.6])
m_ridge.fit(X_train_poly_scaled,y_train)
print(f'Ridge Regression on Training Data R^2: { m_ridge.score(X_train_poly_scaled,y_train)}')
print(f'Ridge Regression on Validation Data R^2: { m_ridge.score(X_val_poly_scaled,y_val)}')
print(f'Linear Regression on Validation Data R^2: {reg.score(X_val, y_val)}')
print(f'Poly Regression on Validation Data R^2: {poly_reg.score(X_val_poly, y_val)}')
print(f'LASSO Regression on Validation Data R^2: { m_lasso.score(X_val_poly_scaled,y_val)}')
print(f'Ridge Regression on Validation Data R^2: { m_ridge.score(X_val_poly_scaled,y_val)}')
print()
print('Choosing Ridge Regression')<jupyter_output>Linear Regression on Validation Data R^2: 0.4204576153833922
Poly Regression on Validation Data R^2: 0.5049139912265036
LASSO Regression on Validation Data R^2: 0.5476677685117314
Ridge Regression on Validation Data R^2: 0.563146272206088

Choosing Ridge Regression<jupyter_text>Model Performance on Testing Data<jupyter_code># print(f'Linear Regression on Test Data R^2: {reg.score(X_test, y_test)}')
# print(f'Poly Regression on Test Data R^2: {poly_reg.score(X_test_poly, y_test)}')
# print(f'LASSO Regression on Test Data R^2: { m_lasso.score(X_test_poly_scaled,y_test)}')
print(f'Ridge Regression on Test Data R^2: { m_ridge.score(X_test_poly_scaled,y_test)}')
y_test_pred = m_ridge.predict(X_test_poly_scaled)

print("MAE:", np.mean(np.abs(y_test_pred-y_test)))
res = y_test_pred-y_test
res = res.reset_index(drop= True)

plt.pyplot.plot(res)
plt.pyplot.savefig('Final Model Errors.svg')
coef_df = pd.DataFrame(list(zip(poly.get_feature_names(list(X_test.columns)),m_ridge.coef_)),columns=['Variable','Coefficient'])
coef_df['Absolute Coefficient'] = np.absolute(coef_df['Coefficient'])

coef_df.sort_values(by='Absolute Coefficient',ascending=False).to_csv('Model Coefficients.csv',index=False)
coef_df.sort_values(by='Absolute Coefficient',ascending=False).head(20)
coef_df[coef_df['Variable'].isin(['Full Art', 'Has EX Burst', 'log(Cost)', 'Ability Count', 'Power','Desc Length', 
           'Normal', 'Hero', 'Legend', 'Promo', 'Rare', 'Starter'])]<jupyter_output><empty_output><jupyter_text>Appendix Check correlation<jupyter_code>X_.corr()[['Full Art', 'Has EX Burst', 'log(Cost)', 'Ability Count', 'Power','Desc Length', 
           'Normal', 'Hero', 'Legend', 'Promo', 'Rare', 'Starter']]<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.14882182720132286, 'fraction_numerical': 0.032658123191401406, 'mean_word_length': 6.938474159146842, 'pattern_counts': {'":': 4, '<': 36, '<?xml version=': 0, '>': 36, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '10', 'dir': 'jupyter-structured-clean-dedup', 'id': '623420', 'license': 'MIT', 'n_tokens_mistral': 3992, 'n_tokens_neox': 3793, 'n_words': 654, 'path': '.ipynb_checkpoints/Data Cleaning and Modelling-checkpoint.ipynb', 'repo_name': 'warren-andrew/lin-reg-tcg', 'size': '9676'}