<jupyter_start><jupyter_text><jupyter_code>#import 
import pandas as pd
import seaborn as sns
from sklearn.impute import SimpleImputer
from sklearn.preprocessing import OrdinalEncoder
import xgboost as xgb 
import matplotlib.pyplot as plt
from sklearn.linear_model import  LinearRegression
from sklearn.svm import SVR
from sklearn.model_selection import train_test_split, cross_val_score
from sklearn.metrics import mean_squared_error
from sklearn.pipeline import make_pipeline
from sklearn.preprocessing import StandardScaler
from sklearn import  metrics
from scipy.stats.stats import pearsonr   
 
!pip install xgboost<jupyter_output>Requirement already satisfied: xgboost in /usr/local/lib/python3.7/dist-packages (0.90)
Requirement already satisfied: numpy in /usr/local/lib/python3.7/dist-packages (from xgboost) (1.19.5)
Requirement already satisfied: scipy in /usr/local/lib/python3.7/dist-packages (from xgboost) (1.4.1)<jupyter_text>**Problem Statement - Variation of scikit's train_test_split random state value with dataset size(normally distributed), for producing better model performance.***Initial experiments to be run on normally distributed datasets with different number of rows, and then in future scope inference to be expanded over to other types of distributions as well*The python code we used to generate normally distributed dataset is in data_generator.pyWe are synthetical generating normally distributed dataset with four attributes.The varying size of data has the same covariance, mean, standard deviation for the attributesSize of dataset used 100, 500, 1000, 10000, 100000Model metrics that are being monitored with different random state values, MSE, cross validaton score for the benchmark models we are testing on(XGB, SVR, LinearRegression). Experimenting with the first dataset mentioned<jupyter_code>df = pd.read_csv('10000rows_dataset.csv')

df.describe()
#checking for normal distribution
sns.distplot(df['col_4']);
#CHECKING FOR CORRELATION BETWEEN THE ATTRIBUTES
corr = df.corr().abs()
sns.heatmap(corr)

#CHECKING FOR NAN VALUES AND IMPUTING THEM
print(df.shape)

df.isnull().sum()


def train_models(df):
    model_linear = LinearRegression()
    model_xgb = xgb.XGBRegressor(n_estimators=5000)
    model_svr = make_pipeline(StandardScaler(), SVR(C=1.0, epsilon=0.2))
    random_states = [i for i in range(10,201,10)]
    #random_states.extend([500, 1000, 5000])
    errors_linear, errors_xgb, errors_svr  = [], [], []
    scores_linear, scores_xgb, scores_svr = [], [], []
    for rs in random_states:

          X_train, X_test, Y_train, Y_test = train_test_split(df.iloc[:,0:-1], df.iloc[:,-1], test_size=0.2, random_state=rs, shuffle = True)
          model_linear.fit(X_train, Y_train)
          Y_linear = model_linear.predict(X_test)
          scores_linear.append(cross_val_score(
          model_linear, df.iloc[:,0:-1], df.iloc[:,-1], cv=5))
          
          model_xgb.fit(X_train, Y_train)
          Y_xgb = model_xgb.predict(X_test)
          scores_xgb.append(cross_val_score(
          model_xgb, df.iloc[:,0:-1], df.iloc[:,-1], cv=5))


          model_svr.fit(X_train, Y_train)
          Y_svr = model_svr.predict(X_test)
          scores_svr.append(cross_val_score(
          model_svr, df.iloc[:,0:-1], df.iloc[:,-1], cv=5))
          
          errors_linear.append(mean_squared_error(Y_test, Y_linear))
          errors_xgb.append(mean_squared_error(Y_test, Y_xgb))
          errors_svr.append(mean_squared_error(Y_test, Y_svr))
    return errors_linear, errors_xgb, errors_svr, scores_linear, scores_xgb, scores_svr



def infer(df, rows):
      random_states = [i for i in range(10,201,10)]
      
      errors_linear, errors_xgb, errors_svr, scores_linear, scores_xgb, scores_svr =train_models(df)
      print("For dataset of rows", rows)
      f, axs = plt.subplots(2,2,figsize=(15,15))
      axs[0,0].plot(random_states, errors_svr[:20])
      axs[0,0].set_xlabel('random_state values')
      axs[0,0].set_ylabel('SVR errors')
      
      # displaying the title
      axs[0,0].set_title('SVR - random_state vs mse error')

      axs[0,1].plot(random_states, errors_linear[:20])
      axs[0,1].set_xlabel('random_state values')
      axs[0,1].set_ylabel('Linear regression errors')
      
      # displaying the title
      axs[0,1].set_title('Linear - random_state vs mse error')

      axs[1,0].plot(random_states, errors_xgb[:20])
      axs[1,0].set_xlabel('random_state values')
      axs[1,0].set_ylabel('XGboost errors')
      
      # displaying the title
      axs[1,0].set_title('XGBoost - random_state vs mse error')
      f.savefig('plots_100')
      print("saving plots")
      plt.show()
   
      print(pearsonr(errors_linear,errors_svr))
      print("Cross validation score for different values of random_state for Linear regression - ", scores_linear)
      print("Cross validation score for different values of random_state for SVR - ", scores_svr)
      print("Cross validation score for different values of random_state for XGBoost - ", scores_xgb)
df = pd.read_csv('100rows_dataset.csv')
infer(df, 100)
infer(df, 500)
infer(df, 1000)
infer(df, 10000)
infer(df, 100000)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1038135593220339, 'fraction_numerical': 0.030046224961479198, 'mean_word_length': 4.198198198198198, 'pattern_counts': {'":': 0, '<': 8, '<?xml version=': 0, '>': 8, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '470797', 'license': 'MIT', 'n_tokens_mistral': 1827, 'n_tokens_neox': 1695, 'n_words': 427, 'path': 'random_state_da.ipynb', 'repo_name': 'jeff10joy/random-state-DA', 'size': '5192'}