<jupyter_start><jupyter_text>Data Preprocessing - Normalization  IntroductionThis code is part of Fuel Leak Detection and Location Project.This code creats file for train data and for test data.  Imports and Global Definitions<jupyter_code>import numpy as np
import seaborn as sns
import pandas as pd
import numpy as np
import matplotlib
import matplotlib.pyplot as plt
from io import StringIO
from IPython.display import clear_output
%matplotlib inline
from statsmodels.graphics.tsaplots import plot_acf
from sklearn.model_selection import train_test_split

label_colors = ["#F58C41", "#2FAFC6", "#800080", "#AF1946", 
                "#46A5E1", "#522A64", "#A3DB05", "#FC6514"]
def select_columns(data_frame, column_names):
    new_frame = data_frame.loc[:, column_names]
    return new_frame<jupyter_output><empty_output><jupyter_text>Data Loading<jupyter_code>#read data file
test_df= pd.read_csv('Test_Data.csv') # Or Train_Data
test_df.head()
neg, pos = np.bincount(test_df['Label'])
total = neg + pos
print('Total: {}\nPositive: {} ({:.2f}% of total)\n'.format(total, pos, 100 * pos / total))
#read data file
train_df= pd.read_csv('Train_Data.csv')
train_df.head()
train_df.shape
neg, pos = np.bincount(train_df['Label'])
total = neg + pos
print('Total: {}\nPositive: {} ({:.2f}% of total)\n'.format(total, pos, 100 * pos / total))<jupyter_output>Total: 2240
Positive: 351 (15.67% of total)<jupyter_text>Normalize Data<jupyter_code>#Normalize the data to 0-100 scale
normal_val = train_df.drop(['File','PT','StartTime','Label','LeakTime'], axis = 1)
points =[x for x in range(0, 2500)]
for index, row in normal_val.iterrows():
    data = row
    maxVal = data.max()
    minVal = data.min()
    normal_val.at[index,'maxVal'] = maxVal
    normal_val.at[index,'minVal'] = minVal

normal_val['h'] = normal_val['maxVal'] - normal_val['minVal']
for x in points:
    point = 's'+str(x)
    normal_val[point] = (normal_val[point] - normal_val['minVal'])/normal_val['h']*100
    if (x % 10) == 1:
        clear_output(wait=True)
        print(x)
selected_columns = ['File','PT','StartTime','Label','LeakTime']
#normal_val.drop(['maxVal','minVal','h'], axis = 1, inplace=True)
normal_df = pd.concat([select_columns(train_df, selected_columns), normal_val], axis=1)
#remove nan or infinite value
normal_df.replace(np.inf, 0)
normal_df.replace(np.nan, 0, inplace=True)

normal_df = normal_df.drop(['maxVal','minVal','h'], axis = 1)
normal_df.head()
normal_df.to_csv('Train_Data_Normal.csv', encoding='utf-8', index=False)
#Normalize the data to 0-100 scale
normal_val = test_df.drop(['File','PT','StartTime','Label','LeakTime'], axis = 1)
points =[x for x in range(0, 2500)]
for index, row in normal_val.iterrows():
    data = row
    maxVal = data.max()
    minVal = data.min()
    normal_val.at[index,'maxVal'] = maxVal
    normal_val.at[index,'minVal'] = minVal

normal_val['h'] = normal_val['maxVal'] - normal_val['minVal']
for x in points:
    point = 's'+str(x)
    normal_val[point] = (normal_val[point] - normal_val['minVal'])/normal_val['h']*100
    if (x % 10) == 1:
        clear_output(wait=True)
        print(x)
selected_columns = ['File','PT','StartTime','Label','LeakTime']
#normal_val.drop(['maxVal','minVal','h'], axis = 1, inplace=True)
normal_df = pd.concat([select_columns(test_df, selected_columns), normal_val], axis=1)
#remove nan or infinite value
normal_df.replace(np.inf, 0)
normal_df.replace(np.nan, 0, inplace=True)

normal_df = normal_df.drop(['maxVal','minVal','h'], axis = 1)
normal_df.head()
normal_df.to_csv('Test_Data_Normal.csv', encoding='utf-8', index=False)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1604529135597901, 'fraction_numerical': 0.02623584645125656, 'mean_word_length': 5.633699633699634, 'pattern_counts': {'":': 0, '<': 12, '<?xml version=': 0, '>': 12, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '247339', 'license': 'Unlicense', 'n_tokens_mistral': 1367, 'n_tokens_neox': 1287, 'n_words': 293, 'path': 'DataPrep - Normalize Data.ipynb', 'repo_name': 'AnatKarny/FuelLeakDetection', 'size': '3621'}