<jupyter_start><jupyter_text>Data Preparation<jupyter_code>" Loading the dataset "
datasets_path = os.path.join(absFilePath, 'Datasets\\')
url = datasets_path + 'data_transcoding_mesurment.tsv'
df = pd.read_csv(url, sep='\t')

" Handling some data "
df = df.drop(columns=['id'])
df = df.rename(columns={'i': 'i_frames', 'p': 'p_frames' , 'b':'b_frames'})

data_df = df.drop(columns=['utime'])
target_df = df['utime']

categorical_feature_mask = (data_df.dtypes == object)
categorical_cols_names = data_df.columns[categorical_feature_mask].tolist()
numerical_cols_names = data_df.columns[~categorical_feature_mask].tolist()

data_df = pd.concat([data_df[numerical_cols_names].astype(float), data_df[categorical_cols_names]],axis = 1)
data_target_df = pd.concat([data_df, target_df], axis=1) 
" generate the Test SET "
nb_test_instances = 1000 
test_df = data_target_df.sample(n=nb_test_instances)
data_test_df = test_df.drop(columns=['utime'])
target_test_df = test_df['utime']

" generate the Training SET "
train_df = pd.concat([data_target_df,test_df]).drop_duplicates(keep=False)
data_train_df = train_df.drop(columns=['utime'])
target_train_df = train_df['utime']

" Extract values of the test set to generate the neighbors"

" Decode Categorical Features "

format_mapper = {
    0 : 'vp8', 1 : 'h264', 2 : 'mpeg4', 3 : 'flv'
}
format_mapper_inv = dict(map(reversed, format_mapper.items()))

data_test_df_copy = data_test_df.copy()
data_test_df_copy['codec']   = data_test_df_copy['codec'].replace(format_mapper_inv)
data_test_df_copy['o_codec'] = data_test_df_copy['o_codec'].replace(format_mapper_inv)

data_test_copy = data_test_df_copy.values
numerical_cols = np.arange(0,len(numerical_cols_names)) 
categorical_cols = np.arange(len(numerical_cols_names),data_df.shape[1])<jupyter_output><empty_output><jupyter_text>Neighbors Generation<jupyter_code># generate neighbors : 
l_nb_neighbors = [20,50,100]
l_list_neigh = []

for nb_neighbors in l_nb_neighbors :
    list_neigh = generate_all_neighbors(data_test_copy,numerical_cols,categorical_cols,nb_neighbors)

    " store all the neighbors together "
    n = np.size(data_test_copy,0)
    all_neighbors = list_neigh[0]
    for i in range(1,n) :
        all_neighbors = np.concatenate((all_neighbors, list_neigh[i]), axis=0)

    " One hot encoding "

    df_neigh = pd.DataFrame(data = all_neighbors,columns= numerical_cols_names + categorical_cols_names)
    df_neigh[categorical_cols_names] = df_neigh[categorical_cols_names].astype(int,errors='ignore')

    " Decode non ordinal features to perform one hot encoding "

    df_neigh['codec']   = df_neigh['codec'].replace(format_mapper)
    df_neigh['o_codec'] = df_neigh['o_codec'].replace(format_mapper)

    " One hot encoding "
    df_neigh = pd.get_dummies(df_neigh, prefix_sep='_', drop_first=True)

    " Store the neighbors in a list"

    data_neigh = df_neigh.values
    n = np.size(data_test_copy,0)
    list_neigh = []
    j = 0
    for i in range(0,n):
        list_neigh.append(data_neigh[j:(j+nb_neighbors),:])
        j += nb_neighbors
    l_list_neigh.append(list_neigh)
list_neigh_20  = l_list_neigh[0] 
list_neigh_50  = l_list_neigh[1]
list_neigh_100 = l_list_neigh[2]<jupyter_output><empty_output><jupyter_text>One hot encoding for the training and the test sets<jupyter_code>data_train_df = pd.get_dummies(data_train_df, prefix_sep='_', drop_first=True)

data_train = data_train_df.values
target_train = target_train_df.values

data_test_df = pd.get_dummies(data_test_df, prefix_sep='_', drop_first=True)
data_test = data_test_df.values
target_test = target_test_df.values<jupyter_output><empty_output><jupyter_text>Training the MLP model<jupyter_code>" Sklearn MLP regressor "

mlp = make_pipeline(StandardScaler(),
                    MLPRegressor(hidden_layer_sizes=(50, 50),
                                 tol=1e-2, 
                                 max_iter=1000, 
                                 random_state=0))
model_nt = mlp.fit(data_train, target_train)
target_pred_nt = model_nt.predict(data_test)<jupyter_output><empty_output><jupyter_text>Execution of Split Based Selection Form Algorithm :<jupyter_code>split_point = len(numerical_cols)
nb_models = 100
(L_Subgroups_20,P_1)  = SplitBasedSelectionForm (data_test, target_test, nb_models, model_nt, list_neigh_20,split_point)
(L_Subgroups_50,P_2)  = SplitBasedSelectionForm (data_test, target_test, nb_models, model_nt, list_neigh_50,split_point)
(L_Subgroups_100,P_3) = SplitBasedSelectionForm(data_test, target_test, nb_models, model_nt, list_neigh_100,split_point)
" Define the functions to save and load data "
import pickle
def save_obj(obj, name):
    with open(name + '.pkl', 'wb') as f:
        pickle.dump(obj, f, pickle.HIGHEST_PROTOCOL)

def load_obj(name):
    with open(name + '.pkl', 'rb') as f:
        return pickle.load(f)
'SAVE THE DATA'

path = './saved_data/'
save_obj(data_train, path + 'data_train_n')
save_obj(target_train, path + 'target_train_n')
save_obj(data_test, path  + 'data_test_n')
save_obj(target_test, path + 'target_test_n')
save_obj(list_neigh_20, path + 'list_neighbors_20')
save_obj(list_neigh_50, path + 'list_neighbors_50')
save_obj(list_neigh_100, path + 'list_neighbors_100')
'SAVE THE LIST OF THE SUBGROUPS'
save_obj(L_Subgroups_20, path  + 'list_subgroups_20')
save_obj(L_Subgroups_50, path  + 'list_subgroups_50')
save_obj(L_Subgroups_100, path + 'list_subgroups_100')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11723631018008085, 'fraction_numerical': 0.018375597206909223, 'mean_word_length': 5.820802005012531, 'pattern_counts': {'":': 0, '<': 21, '<?xml version=': 0, '>': 21, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 3, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '369537', 'license': 'MIT', 'n_tokens_mistral': 2163, 'n_tokens_neox': 2070, 'n_words': 334, 'path': 'tabular data/regression/Benchmarks/3. transcoding videos/supplementary tests/trans_nb_neighs.ipynb', 'repo_name': 'RemilYoucef/SPLITSD4X', 'size': '5442'}