<jupyter_start><jupyter_text>Imports<jupyter_code># General
import pandas as pd
import numpy as np
from tqdm.notebook import tqdm
import random, pickle

# Visualisation
from mpl_toolkits.mplot3d.axes3d import Axes3D
import matplotlib.pyplot as plt
from matplotlib import cm
import seaborn as sns
from matplotlib import animation

# Algos<jupyter_output><empty_output><jupyter_text>Declaring Paths and Initialising Variables<jupyter_code>root_path = '../'
raw_datapath = root_path+'Raw Data/'
prepared_datapath = root_path+'Notebook Results/'
housing_datapath = raw_datapath+'boston_data.csv'
housing_testdatapath = raw_datapath+'boston_test_data.csv'


'''
/kaggle/input/boston-dataset/boston_test_data.csv
/kaggle/input/boston-dataset/boston_data.csv
'''
target_variable = 'medv'<jupyter_output><empty_output><jupyter_text>Reading the Dataset- **crim** per capita crime rate by town.- **zn** proportion of residential land zoned for lots over 25,000 sq.ft.- **indus** proportion of non-retail business acres per town.- **chas** Charles River dummy variable (= 1 if tract bounds river; 0 otherwise).- **nox** nitrogen oxides concentration (parts per 10 million).- **rm** average number of rooms per dwelling.- **age** proportion of owner-occupied units built prior to 1940.- **dis** weighted mean of distances to five Boston employment centres.- **rad** index of accessibility to radial highways.- **tax** full-value property-tax rate per 10,000.- **ptratio** pupil-teacher ratio by town.- **black** 1000(Bk - 0.63)^2 where Bk is the proportion of blacks by town.- **lstat** lower status of the population (percent).- **medv** median value of owner-occupied homes in $1000s.<jupyter_code>housing_data = pd.read_csv(housing_datapath, index_col=0).reset_index()
housing_testdata = pd.read_csv(housing_testdatapath, index_col=0)<jupyter_output><empty_output><jupyter_text>Visualisation<jupyter_code>plt.rcParams['figure.dpi'] = 180
plt.rcParams['figure.figsize'] = (5,5)

class TestingClass:
    
    def __init__(self):
        print('Initialising Testing Class...')
    
    def create_bootstrappeddata(self, df, targetcol, newsize, _features, std_scaling=0.1, mode='ot'):

        df = df[_features].dropna().reset_index().drop('index', axis=1)

        bootstrappeddata = pd.DataFrame()

        for _ in range(newsize):
            temp = df.sample(1)
            if mode == 'ot':
                topertub = [targetcol]
            elif mode == 'of':
                topertub = [k for k in _features if k!=targetcol]
            elif mode == 'all':
                topertub = _features

            for eachFeat in topertub:

                _std = df[eachFeat].dropna().std()*std_scaling
                origval = temp[eachFeat].values[0]
                possiblevals = np.linspace(origval-_std, origval+_std, 20).tolist()
                temp[eachFeat] = random.sample(possiblevals, k=1)
                bootstrappeddata = pd.concat([bootstrappeddata, temp])

        return bootstrappeddata

testingClassHandler = TestingClass()

var_x = 'lstat'
var_y = 'indus'
var_z = target_variable
pltData = housing_data[[var_x, var_y, var_z]].dropna()
_mode = 'of'
datadict = {}

for eachmode in ['ot', 'of', 'all']:
    datadict[eachmode] = {}
    for eachstd in tqdm(np.linspace(0, 0.5, 101)):
        
        datadict[eachmode][eachstd] = testingClassHandler.create_bootstrappeddata(df = pltData, 
                                                                                  targetcol = target_variable,
                                                                                  newsize = 1000,
                                                                                  _features = pltData.columns.tolist(),
                                                                                  std_scaling = eachstd,
                                                                                  mode=eachmode)

df = pd.DataFrame()
for eachP1key in tqdm(datadict.keys()):
    for eachP2Key in datadict[eachP1key].keys():
        tempdf = datadict[eachP1key][eachP2Key].copy()
        tempdf['mode'] = eachP1key
        tempdf['stdscalig'] = eachP2Key
        df = pd.concat([df, tempdf])

df.to_csv(prepared_datapath+'prep_vizdata.csv')
unique_col_set = df.groupby(['mode', 'stdscalig']).size().index.tolist()
modedecode = {'ot':'Only Target', 'of':'Only Features', 'all':'Features & Target'}
def update_graph(num):
    _select = list(unique_col_set[num])
    data=df[df[['mode', 'stdscalig']].isin(_select).product(axis=1) == 1]
    f1corr = np.round(data[[var_x, var_y, var_z]].corr()[[var_z]].iloc[0].values[0], 3)
    f2corr = np.round(data[[var_x, var_y, var_z]].corr()[[var_z]].iloc[1].values[0], 3)
    graph.set_edgecolor('k')
    graph.set_color('white')
    graph._offsets3d = (data.lstat, data.indus, data.medv)
    title.set_text('Data Perturbation, Mode: {0}, \nStdScaling: {1}; \nF1 Corr : {2}, F1 Corr : {3}\n\n'.format(modedecode[_select[0]], 
                                                                                               np.round(_select[1],3),
                                                                                               f1corr,
                                                                                               f2corr))
    
    if _select[0] == 'all':
        if num > 15:
            ax2.view_init(elev=10, azim=num-35)
        else:
            ax2.view_init(elev=10, azim=0)
    elif _select[0] == 'of':
        if num > 115:
            ax2.view_init(elev=10, azim=num-135)
        else:
            ax2.view_init(elev=10, azim=0)
    elif _select[0] == 'ot':
        if num > 215:
            ax2.view_init(elev=10, azim=num-235)
        else:
            ax2.view_init(elev=10, azim=0)
    
    
fig = plt.figure(figsize=plt.figaspect(0.5), facecolor='lightgray')

data = pltData.copy()
ax1 = fig.add_subplot(1, 2, 1, projection='3d', facecolor='lightgray')
# ax1.view_init(elev=10, azim=0)
ax1.scatter(data.lstat, data.indus, data.medv, c='white', edgecolor='k')
ax1.set_title('Actual\n')
ax1.set_xlabel('Feature 1', rotation=90)
ax1.set_ylabel('Feature 2', rotation=90)
ax1.set_zlabel('Target Variable', rotation=90)
ax1.w_xaxis.set_pane_color((0.0, 0.0, 0.0, 0.0))

ax2 = fig.add_subplot(1, 2, 2, projection='3d', facecolor='lightgray')
ax2.set_title(f'BootStrapped - Mode : all')     
data=df[df['mode']=='all'].copy()
data=data[data['stdscalig']==0.0]
graph = ax2.scatter(data.lstat, data.indus, data.medv, c='white', edgecolor='k')
title = ax2.set_title('Data Perturbation, Mode: 0')
ax2.set_xlabel('Feature 1', rotation=90)
ax2.set_ylabel('Feature 2', rotation=90)
ax2.set_zlabel('Target Variable', rotation=90)
ax2.view_init(elev=10, azim=0)
ani = animation.FuncAnimation(fig, update_graph, interval=500, blit=False,
                              repeat=True, save_count=303)

plt.show()
FFwriter = animation.FFMpegWriter(fps=3, codec="libx264")     
ani.save('basic_animation1.mp4', writer = FFwriter)
# %matplotlib inline<jupyter_output><empty_output><jupyter_text>Random Testing Space<jupyter_code>%matplotlib notebook
# plt.rcParams["animation.html"] = "jshtml"
plt.rcParams['animation.embed_limit'] = 2**128
from IPython.display import HTML
HTML(ani.to_jshtml())
FFwriter = animation.FFMpegWriter(fps=3, codec="libx264")     
ani.save('basic_animation1.mp4', writer = FFwriter)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1337562424078823, 'fraction_numerical': 0.02807396409771899, 'mean_word_length': 2.693918245264207, 'pattern_counts': {'":': 0, '<': 21, '<?xml version=': 0, '>': 24, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 27, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '654191', 'license': 'MIT', 'n_tokens_mistral': 2560, 'n_tokens_neox': 2414, 'n_words': 490, 'path': 'Pedulum Simulation/Notebooks/.ipynb_checkpoints/Handling Small Datasets-checkpoint.ipynb', 'repo_name': 'ag-ds-bubble/medium_blog_open', 'size': '7409'}