<jupyter_start><jupyter_text>Cost Function Visualizations  Load and prepare the data<jupyter_code>import numpy as np
import pandas as pd<jupyter_output><empty_output><jupyter_text>Lets load the dataset unto our workspace with Pandas:<jupyter_code>df = pd.read_csv('weatherww2.csv')
df = df[['MinTemp', 'MaxTemp']]
df.head(10)<jupyter_output>/home/vanillaskies/.local/lib/python3.8/site-packages/IPython/core/interactiveshell.py:3145: DtypeWarning: Columns (7,8,18,25) have mixed types.Specify dtype option on import or set low_memory=False.
  has_raised = await self.run_ast_nodes(code_ast.body, cell_name,<jupyter_text>This is a pretty straightforward and simple linear regression task<jupyter_code>df.plot.scatter('MinTemp', 'MaxTemp')<jupyter_output><empty_output><jupyter_text>Setting up the visualizationsLet's visualize for this data how the cost function will look like for the MSE loss for different parameters and models<jupyter_code># define the vectorized MSE cost function 
def mse_cost(predictions, target):
    N = predictions.shape[0]
    diff = predictions.ravel() - target.ravel()
    cost = np.dot(diff, diff.T) / N
    return cost
# define the prediction for a simple linear model
def LinearModel(thetas, X):
    # normalize add bias term
    X = (X - X.mean()) / X.std()
    X = np.hstack((X, np.ones((X.shape[0], 1), dtype=X.dtype)))
    return np.dot(X, thetas)
# initialize data structures
vis_df = df.sample(frac=0.1)
y = vis_df.iloc[:, 1].to_numpy().reshape(-1, 1)
X = vis_df.iloc[:, 0].to_numpy().reshape(-1, 1)
# grid search over "all" possible theta values and compute cost
start, end, step = -200, 200, 5
thetas_0, thetas_1 = np.arange(start, end, step), np.arange(start, end, step)

# loop over the all the parameter pairs and create a list of all possible pairs
thetas_lst = []
for theta_0 in thetas_0:
    for theta_1 in thetas_1:
        thetas_lst.append(np.array([theta_0, theta_1]).reshape(-1, 1))

linear_cost_lst = []       
for thetas in thetas_lst:
    # get prediction from our model
    pred_linear = LinearModel(thetas, X)
    # keep track of the cast per parameter pairs
    linear_cost_lst.append(mse_cost(pred_linear, y))<jupyter_output><empty_output><jupyter_text>Linear Model Cost Function Visualization![plotly logo](data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAWcAAACMCAMAAACXkphKAAAAwFBMVEX///8Rnf9QZ4QAl/8l/v3L5P8Am//Y6v9ktf8Alv9GX35JYoA6VnhCXXyTn6/O09qxucSNmqvz9Pbo8v+02P+AwP/l6Otvuf9qfJRTaocYv/719vgg5P7t7/KstcK/xtATpv+cp7Yj8/3K0NhgdY+Aj6Mf3v4d1f7Z3eN0hZsayP4e2f4YvP4k+P0Uqv9tf5e43f8Ws/4h6v0bz/4f4P4Xtv+Kx/9Nr//v+P+22f8i8P0Vrv+AxP+k1P87qv/C4P/M3M6FAAAJ4klEQVR4nO2da0MaOxCGFxa2shdArSgiAgVarFpsT2s9ba3//1+dveWyySS7BAirZ95vZtPAPkwmk8mljoNCoVAoFAqFQqFQKBQKhUKhUCgUCoUyUHvPOj30C9ZCDw133zoaHfolD6+XjtfYt7y/h37Lw6uxf8yNhvt86Nc8tN65FjA3vPNDv+ehZYnz0aHf89BCznaEnO0IOdsRcrYj5GxHyNmOkLMdIWc7Qs52hJztCDnbEXK2I5CzlyTnvSpFQlIVKELOqSDOXuPz96tPU/6JO/109f1zIVftfr29+njJY/Xcy+9Xt7+hFnfBedTvdvtbrcysZt1ZT1dh3u3OBtt8glIAZ29630p0yR65d2nJ/ZRRdT+lRVceLfIa39Kif4Amd8B5MIyCIBpugWEW+XELx+oKy/gT/Khv/glqQZw/tDJ9IQi9L3nJB0rV/ZwXfaQNdK7yokvJdeyA86gZNGMFQ2OL7odJA83oWlVh6acVQq3JG0rm7F3muFrfyTP3Y0tE6N2TImLj9MdofZDb3J7zddTMMBmb27CZS/FLDbLfoRlcmH6CRjLn3CHwvDrEwlu3LvERpKR1RzjT36e1D87HQU5B0++1OskxNiOF6+nlv2QzNP6Samk5d6pw/mGZc9ewgbpxrqnfeHOcazoOvkHOeVx3V6u47u1xtjdPOen12DsPVr1eb6WM2so5j9IGlPE15exPkoqpVnwFkPNqvnJU0j0Tdch59yyKdXOSIOrfxBOERP5CMSEr4byaLfIGguUE/LEo52ZWMVUwZxUAzr1hXGcIx9O6Z7IOmEeape8VLJyTbhAFBEIziNbQl9dy7q0LDQTdE7kO48wrZKBlzqvsn4SQ2a4i9TNAh+NMpwU3flB89yC8GUvVNZzH61BsAZjNwJybQw3nRdZsbAqyLjTPasW57xMo8ssHfH8u4zwJgBaiG9F5KDizUVHiPCYFkWy0K/LMr5ZwqQFnUKGYhVByvob5BUOhT6g4U4YSZ9LjmsFS+vbLQPr3r5KzlO5RcSZ5Dxl0swhawdlX2/MJ7SeR6MfG9MsHsot7VZyb0aQK54kKc+I6C64D5hysaQXZP1Oj9WfCl59Rpyeb+ivjLBgRzHmgxhzTKTAAOfucdwHiDda64O2ZpW8Tb3hup9MpPvDikuIJi6xSMVoGiipz9qMwjLjwLA5Eyjmv+fpxA2HEtxryoynjnNTLFB1zASAQP9NUqtC5WC+qGG6AnN0f8QT6/jP/xHt8iuuec0Ve4zaZdk/5+eDftuM8PRvZc7ToD0aj8XzJkS68HMiZ8xpBtJyPR6NVf8GZ+JCry+bdCo8KcFbxvKCj4ERspTpn91LMEDXch6z2GZfM+JWlPL6wop95k/KRl1LOPo13BxesnMcEch6yBtYDoGEeg0l+Y8QMmvcPzJ8EVVd3gLzoVE6C0uMlU0qepEp/0Qbcp7zSc6W8aIEz/+rH9EHE9XuIMzO3wqrfgHUJ7pcyyiPB4516fFQKyPOTjGfrG80/t0n1R2qqNKn/2xNZPm3K2S8OJjcB8HIQZ1ax4MqdFful2BTeiPOY/ZLM24xDoLBE1dat6InWfzXrVme00Q05+0KoPGLdkg1TAGcWxPpC972mZshqm+VFqekGzHQ3DuocPedfdN2Kcn6vWbcy5twUH1FMnOMAOFO3If5QXNzFxi8zzswV04V2hdPWC/DPXwnBT7I9v6eu5Bup1diWs0yJGipnRADnGbU1qftSkwtpjzDM8y8oVDKmbh7UOXC8kQ9x93SpBODs/SaDJS0x5Qy8N3G8nN8FOK/lWkTUDJmDNuTMqJIxVSZfQWD8fJuOgiw0Bjg33K/3BczmnIGdL9RSWbwAcCb9N5BH/RPCwqcsDDmPSEvESwCepILA+aA7vbv8zc3+IM7xfPDH5V2D+9uUM9D7qINmrhvgTB0PsHVmKD0zXR8URz2DoM5R5Tc8z+PnGiBnsZYxZ2D7z2QjzlD/JZ17e84stEtTe+zPsHJQ51TMIyk4CywNOUPbrJhTpEU6ztKiwC45C6GdSVDnIGdOKs7MIUeAu64o5Eyl3L9Bs0bxoGoU1DnImZOSM8/WKKhzkDMn9X4kNr+n3rkZwI2ohJyp1Jw5uow43IhKGNdRqTmP5Y0L/iZBnaOcp5yd/eywvxXzlKOzcy6l/1bnKYmWIujNgjpHMe9OV0/e6efd3lGa1v/zCufdij1xGs7cgmzexkZBnQPnkfJFqidtHilfpHL+0EWXmueRaFYbMv9Eun25F0WD3vwIC5AXJQSdZx3nd6ToteRFmfkr+ryOs7BNZMOgzgE50zv9TnV5ftrC0SvJ87NVrhDYTeqU7DNnK77plwAb0AlYT3lPnlXjfP5K1q24QvjIlpYzG5mbmwd1Tj0421mH5fMUQMBdwnnMO45NgzqnJpyt7Cvg+z4IWn8+5Zg1unFQ59SGs4V9MvEDZpPRTA4m9Zy50K7qHn5eNeFsYd8Xv1CdbGCcrYThsOS81RoaWyurLpz3v48x1pzfMppUL9vHyMs0U5epPpwl7XhfbqKl+Jn6fbm86NONll+p6st51/vME40WUqJCt8+cF1t+Vd4roVNtOe/83ESqsXTqSHNugteA7amDZzklqivn3Z8DyjQYCqA154Cgjze9l6KenIFzbd1tz7XlGi2FVIVsz5H8r+YsUjcI6pxacB6KnMBzmsQRA1OMauc02QcXYhIW+o3IwVF5GjJgCyqGl80cnnM8ElU5d+zcpKAjcC5W5dwxp9mQVudDv0kKGvDr3GALT9nLVQPOsYWMJqXn6GMXmTxXecfyc/S8RvPj7DR90T9NkrKF+PnzG87TmMxREtWDc6KSeyFijXs9XQKn5F4IqbWVcC9E1kTxqojZcl2YO5mac40411DdMCiOHSYZpEzIWS0pNN/i8jzkrJYUeFa8mwASclZKzJ0EgVnonAo5KyVw9hebr6IwAeuwlNcLPfTzQIrYLgJyKJMd3aT/+d1pJc5k+mV8weLexfuNIDK9ZywTwLlBrPcvhUoQntL9R95jXvRAG+iQ05xnntQmdD9Svici3M9FwDtQn2ZO4hnqFj4jEXh/XWarHK784PETf39dtv2gzd9fl+3pkI8dw5xPFsk1wME+bqfdkdKLjP3IH3a3tgX4/rrHdvv5L2+V3vS53X4s3l939NB+OSveX/fnpf3+J9QifH/dpNu93sbp7V2rWbc7m+yiw8H3b3jyNXRJEd4bbyz8fxDsCDnbEXK2I+RsR8jZjpCzHSFnO0LOdoSc7Qg52xFytiPkbEfI2Y4scT4v/yZvXPKlq3uQ+1z+Rd64Xjr7B03Xvf7Pemi4+9bRVv/b6JtRe886Lf8KKBQKhUKhUCgUCoVCoVAoFAqFQqFQqP+T/gO2mBD/HZDhkwAAAABJRU5ErkJggg==)An open-source, interactive graphing library for Python<jupyter_code># arrange the costs back to a square matrix grid
axis_length = len(np.arange(start, end, step))
linear_cost_matrix = np.array(linear_cost_lst).reshape(axis_length, axis_length)
import plotly.graph_objects as go

# plot the surface plot with plotly's Surface
fig = go.Figure(data=go.Surface(z=linear_cost_matrix,
                                x=thetas_0,
                                y=thetas_1))

# add a countour plot
fig.update_traces(contours_z=dict(show=True, usecolormap=True,
                                  highlightcolor="limegreen", project_z=True))

# annotate the plot
fig.update_layout(title='Linear Model MSE Cost Surface',
                  scene=dict(
                    xaxis_title='theta_0 (intercept)',
                    yaxis_title='theta_1 (slope)',
                    zaxis_title='MSE Cost'),
                  width=700, height=700)

fig.show()<jupyter_output><empty_output><jupyter_text>Neural Net Model Cost Function Visualization<jupyter_code># define the predictions function for the mini-network
def MiniNet(thetas, X):
    z1 = X * thetas[0]
    a1 = np.tanh(z1)
    z2 = a1 * thetas[1]
    return z2<jupyter_output><empty_output><jupyter_text>Behold! The all Powerful MiniNet ~![MiniNet](https://i.ibb.co/0tC7gLQ/MiniNet.png)<jupyter_code># repeat the process for the NN

nn_cost_lst = []       
for thetas in thetas_lst:
    # get prediction from our model
    pred_nn = MiniNet(thetas, X)
    # keep track of the cast per parameter pairs
    nn_cost_lst.append(mse_cost(pred_nn, y))
# arrange the costs back to a square matrix grid
nn_cost_matrix = np.array(nn_cost_lst).reshape(axis_length, axis_length)
# plot the surface plot with plotly's Surface
fig = go.Figure(data=go.Surface(z=nn_cost_matrix,
                                x=thetas_0,
                                y=thetas_1))

# add a countour plot
fig.update_traces(contours_z=dict(show=True, usecolormap=True,
                                  highlightcolor="limegreen", project_z=True))

# annotate the plot
fig.update_layout(title='MiniNet Model MSE Cost Surface',
                  width=700, height=700)

fig.show()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.07427184466019418, 'fraction_numerical': 0.08895631067961166, 'mean_word_length': 7.103244837758112, 'pattern_counts': {'":': 0, '<': 28, '<?xml version=': 0, '>': 28, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '7', 'dir': 'jupyter-structured-clean-dedup', 'id': '241887', 'license': 'MIT', 'n_tokens_mistral': 4618, 'n_tokens_neox': 4307, 'n_words': 381, 'path': '_posts/cs229/.ipynb_checkpoints/Cost Visualizations - Plotly-checkpoint.ipynb', 'repo_name': 'psymbio/psymbio.github.io', 'size': '8240'}