<jupyter_start><jupyter_text>Fetching records of each vehicle type<jupyter_code>train_data = dict()
for vehicle_type in vehicle_types:
    train_data[vehicle_type] = TraverseFiles(train_path, vehicle_type)
for vehicle_type in vehicle_types:
    print(train_data[vehicle_type].shape)<jupyter_output>(2066103, 14)
(7619485, 14)
(647055, 14)
(121510, 14)
(495118, 14)
(1091578, 14)
(69295, 14)<jupyter_text>Basic stats before excluding idling records  Min Max Normalisation<jupyter_code>min_max_tbl = dict()

cols = train_data[vehicle_types[0]].shape[1]-1

for vehicle_type in vehicle_types:
    data = train_data[vehicle_type]
    print(vehicle_type,'stats:')
    vehicle_col_mm = list()
    for i in range(cols):
        min_max = dict()
        min_max['min'] = np.min( data[:, i])
        min_max['max'] = np.max( data[:, i])
        print('\t\tmin:', min_max['min'])
        print('\t\tmax:', min_max['max'])
        vehicle_col_mm.append(min_max)
    min_max_tbl[vehicle_type] = vehicle_col_mm
def CreateDir(dir_name):
    if not os.path.exists(dir_name):
        os.makedirs(dir_name)

def Normalise(path, export_path, vehicle_type, mm_list):

#vehicle_type: one string element under vehicle_types = ['ZVe44', 'ZV573', 'ZV63d', 'ZVfd4', 'ZVa9c', 'ZVa78', 'ZV252']
    path = path + vehicle_type
    export_path = export_path+vehicle_type
    CreateDir(export_path)
    #these are variables to calculate traversing progress (DO NOT CHANGE)
    counts_per_percent = int(len(os.listdir(path)) / 100)
    percentage_completion = 0
    counter = 0

    for file in os.listdir(path):
        sample_df = pd.read_csv(path + '/' + file, delimiter = ',', encoding = 'utf-8')
        rows = sample_df.shape[0]
        for i in range(len(mm_list)):
            f_min = mm_list[i]['min']
            f_max = mm_list[i]['max']
            diff = f_max - f_min
            if diff == 0:
                continue
            for n in range(rows):
                v = sample_df.iloc[n,i]
                sample_df.iloc[n, i] = (v - f_min) / diff
        sample_df.to_csv (export_path+'/'+file, index = False)

        
        #belows are to show traversing progress (DO NOT CHANGE)
        counter += 1
        if counter == counts_per_percent:
            counter = 0
            percentage_completion += 1
            print('normalising data under', path, ':', percentage_completion, "%", end="\r", flush=True)
for vehicle_type in vehicle_types:
    Normalise(train_path, train_cleaned_path, vehicle_type, min_max_tbl[vehicle_type])
    Normalise(test_path, test_cleaned_path, vehicle_type, min_max_tbl[vehicle_type])<jupyter_output>normalising data under ../data/final/train/ZV252 : 115 %
{'fraction_non_alphanumeric': 0.12350746268656716, 'fraction_numerical': 0.034328358208955224, 'mean_word_length': 3.2896, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '220498', 'license': 'MIT', 'n_tokens_mistral': 1003, 'n_tokens_neox': 918, 'n_words': 203, 'path': 'Iterations/train-test_split/Normalisation.ipynb', 'repo_name': 'BasemSuleiman/Intelligent_Failure_Prediction', 'size': '2680'}