<jupyter_start><jupyter_text>Clean data from test1_1.txt to dataframe type - This program is used to export the txt file to dataframe in python. - The txt is captured from the experiment1 in Github. - File location : https://github.com/TNatapon/Privacy_SDN_Edge_IoT/blob/main/PlanB/Experiment1/test1_1.txt<jupyter_code># import library
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import requests
import io<jupyter_output><empty_output><jupyter_text>File name that we want to clean<jupyter_code>url = 'https://github.com/TNatapon/Privacy_SDN_Edge_IoT/blob/main/PlanB/Experiment1/test1_1.txt'
filename = "/Users/kalika/Desktop/test1_1.txt"
skip_1 = list(range(0, 10, 1))
df = pd.read_csv(filename,
   skiprows=skip_1,
   header=None,
   delimiter=' ',
   skipinitialspace=True,
   error_bad_lines=False)
print(df)
# Keep results of the 1st itteration in the dataframe and drop the results of 2nd itteration and so on ...
df1 = df.drop(labels=range(30, len(df)), axis=0)
print(df1.head)
print(df1.shape)
# Replace new columns header 
df1 = df1.drop(df1.columns[[0,1,3, 5,7,9,11]], axis=1)  
df1.rename({2 :'Interval', 4 : 'Transfer', 6 :'Bitrate', 8 :'Jitter', 10 :'Lost/Total Datagrams'}, axis=1, inplace=True)

print('Iterration 1: \n',df1)<jupyter_output>Iterration 1: 
        Interval Transfer Bitrate Jitter Lost/Total Datagrams
0     0.00-1.00     3.96    33.2  0.509               0/2867
1     1.00-2.00     3.99    33.5  0.636               0/2889
2     2.00-3.00     3.94    33.1  0.722               0/2856
3     3.00-4.00     4.00    33.6  0.505               0/2897
4     4.00-5.00     3.95    33.2  0.757               0/2862
5     5.00-6.00     3.96    33.2  0.608               0/2865
6     6.00-7.00     3.95    33.1  0.670               0/2858
7     7.00-8.00     3.99    33.4  0.568               0/2886
8     8.00-9.00     3.89    32.6  0.523               0/2815
9    9.00-10.00     3.97    33.3  0.566               0/2877
10  10.00-11.00     3.95    33.1  0.442               0/2858
11  11.00-12.00     4.01    33.6  0.511               0/2903
12  12.00-13.00     3.97    33.3  0.558               0/2873
13  13.00-14.00     3.97    33.3  0.408               0/2878
14  14.00-15.00     3.96    33.2  0.467               0/2865
15  15.0[...]<jupyter_text>df2: Itteration 2<jupyter_code># df2 : Itteration 2

df2 = df.drop(labels=range(0, 39), axis=0)
print(df2)
print(df2.head)
# Keep results of the 1st itteration in the dataframe and drop the results of 2nd itteration and so on ...
df2t = df2.drop(labels=range(68, len(df)), axis=0)
print(df2t.head)
# Replace new columns header 
df2t = df2t.drop(df2t.columns[[0,1,3, 5,7,9,11]], axis=1)  
df2t.rename({2 :'Interval', 4 : 'Transfer', 6 :'Bitrate', 8 :'Jitter', 10 :'Lost/Total Datagrams'}, axis=1, inplace=True)

print('Iterration 2: \n',df2t)<jupyter_output>Iterration 2: 
        Interval Transfer Bitrate Jitter Lost/Total Datagrams
39    1.00-2.00     3.92    32.9  0.537               0/2837
40    2.00-3.00     4.00    33.5  0.396               0/2893
41    3.00-4.00     3.95    33.2  0.695               0/2862
42    4.00-5.00     3.99    33.5  0.510               0/2888
43    5.00-6.00     4.01    33.7  0.601               0/2905
44    6.00-7.00     3.99    33.4  0.842               0/2886
45    7.00-8.00     3.98    33.4  0.361               0/2880
46    8.00-9.00     3.96    33.3  0.576               0/2871
47   9.00-10.00     3.97    33.3  0.607               0/2875
48  10.00-11.00     4.00    33.6  0.526               0/2898
49  11.00-12.00     3.98    33.4  0.547               0/2882
50  12.00-13.00     3.95    33.1  0.692               0/2857
51  13.00-14.00     3.99    33.5  0.799               0/2888
52  14.00-15.00     4.00    33.5  0.608               0/2893
53  15.00-16.00     3.96    33.2  0.560               0/2869
54  16.0[...]<jupyter_text>df3 : Itteration 3<jupyter_code>df3 = df.drop(labels=range(0, 77), axis=0)
print(df3.head)
# Keep results of the 1st itteration in the dataframe and drop the results of 2nd itteration and so on ...
df3t = df3.drop(labels=range(106, len(df)), axis=0)
print(df3t.head)
print('Iterration 2: \n',df3t)<jupyter_output>Iterration 2: 
     0   1            2    3     4       5     6          7      8   9   \
77   [  5]    1.00-2.00  sec  3.63  MBytes  30.5  Mbits/sec  0.826  ms   
78   [  5]    2.00-3.00  sec  3.92  MBytes  32.9  Mbits/sec  0.411  ms   
79   [  5]    3.00-4.00  sec  3.99  MBytes  33.5  Mbits/sec  0.807  ms   
80   [  5]    4.00-5.00  sec  3.96  MBytes  33.2  Mbits/sec  0.543  ms   
81   [  5]    5.00-6.00  sec  3.97  MBytes  33.3  Mbits/sec  0.514  ms   
82   [  5]    6.00-7.00  sec  3.98  MBytes  33.4  Mbits/sec  0.626  ms   
83   [  5]    7.00-8.00  sec  3.95  MBytes  33.2  Mbits/sec  0.528  ms   
84   [  5]    8.00-9.00  sec  3.98  MBytes  33.3  Mbits/sec  0.506  ms   
85   [  5]   9.00-10.00  sec  3.96  MBytes  33.2  Mbits/sec  0.549  ms   
86   [  5]  10.00-11.00  sec  3.99  MBytes  33.5  Mbits/sec  0.595  ms   
87   [  5]  11.00-12.00  sec  3.96  MBytes  33.2  Mbits/sec  0.780  ms   
88   [  5]  12.00-13.00  sec  4.02  MBytes  33.7  Mbits/sec  0.678  ms   
89   [  5]  13.00-14.0[...]<jupyter_text>df4 Itteration 4<jupyter_code>df4 = df.drop(labels=range(0, 115), axis=0)
print(df4.head)
# Keep results of the 1st itteration in the dataframe and drop the results of 2nd itteration and so on ...
df4t = df4.drop(labels=range(145, len(df)), axis=0)
print(df4t.head)
print('Iterration 2: \n',df4t)<jupyter_output>Iterration 2: 
     0   1            2    3     4       5     6          7      8   9   \
115  [  5]    1.00-2.00  sec  3.59  MBytes  30.1  Mbits/sec  0.557  ms   
116  [  5]    2.00-3.00  sec  3.71  MBytes  31.1  Mbits/sec  0.486  ms   
117  [  5]    3.00-4.00  sec  3.71  MBytes  31.1  Mbits/sec  0.688  ms   
118  [  5]    4.00-5.00  sec  3.90  MBytes  32.7  Mbits/sec  0.488  ms   
119  [  5]    5.00-6.00  sec  3.92  MBytes  32.9  Mbits/sec  0.595  ms   
120  [  5]    6.00-7.00  sec  3.98  MBytes  33.4  Mbits/sec  0.502  ms   
121  [  5]    7.00-8.00  sec  3.96  MBytes  33.2  Mbits/sec  0.634  ms   
122  [  5]    8.00-9.00  sec  3.91  MBytes  32.8  Mbits/sec  0.668  ms   
123  [  5]   9.00-10.00  sec  3.84  MBytes  32.2  Mbits/sec  0.494  ms   
124  [  5]  10.00-11.00  sec  3.89  MBytes  32.7  Mbits/sec  0.466  ms   
125  [  5]  11.00-12.00  sec  4.00  MBytes  33.6  Mbits/sec  0.564  ms   
126  [  5]  12.00-13.00  sec  3.94  MBytes  33.1  Mbits/sec  0.720  ms   
127  [  5]  13.00-14.0[...]<jupyter_text>df5 Itteration 5<jupyter_code>df5 = df.drop(labels=range(0, 153), axis=0)
print(df5.head)
# Keep results of the 1st itteration in the dataframe and drop the results of 2nd itteration and so on ...
df5t = df5.drop(labels=range(183, len(df)), axis=0)
print(df5t.head)
# Replace new columns header 
df5 = df5t.drop(df5t.columns[[0,1,3, 5,7,9,11]], axis=1)  
df5.rename({2 :'Interval', 4 : 'Transfer', 6 :'Bitrate', 8 :'Jitter', 10 :'Lost/Total Datagrams'}, axis=1, inplace=True)

print(df5.head)
## function to get dataframe from df 

def getDataframeThru(df,start_row,measurement_interval,header_range):
    '''
    measurement_interval = 30 (sec) : 
    header_range = 9 lines
    start_row = 0 
    '''
    
    df1 = df.drop(labels=range(start_row, header_range), axis=0)
    df1t = df1.drop(labels=range(measurement_interval, len(df)), axis=0)
    
    return df1t
skip_1 = list(range(0, 10, 1))
df = pd.read_csv(filename,
   skiprows=skip_1,
   header=None,
   delimiter=' ',
   skipinitialspace=True,
   error_bad_lines=False)
print(df)
h = 9
measurement_interval = 30 
 
start_row = 0 
df_all = df.drop(labels=range(1, len(df)), axis=0)
print(df_all)
for i in range(0,2):
    print('i = ',i)
    if i == 0 :
        m = 30 + h
        df1 = getDataframeThru(df,start_row,m,0)
        result = pd.concat([df_all,df1])
        df_all = df.drop(labels=0, axis=0)
    else:
        header_range = measurement_interval + header_range -1
        measurement_interval = measurement_interval + header_range -1
        print('measurement_interval: ',measurement_interval)
        df1 = getDataframeThru(df,start_row,measurement_interval,header_range)
        print('df1 = ',df1)
        result = pd.concat([df_all,df1])
        print('hd = ', header_range)
    df_all = result

print(df_all)
print('df1=',df1)<jupyter_output>0           1   \
0                                                   [          5]   
0                                                   [          5]   
1                                                   [          5]   
2                                                   [          5]   
3                                                   [          5]   
4                                                   [          5]   
5                                                   [          5]   
6                                                   [          5]   
7                                                   [          5]   
8                                                   [          5]   
9                                                   [          5]   
10                                                  [          5]   
11                                                  [          5]   
12                                [...]
{'fraction_non_alphanumeric': 0.11819645732689212, 'fraction_numerical': 0.15877616747181963, 'mean_word_length': 1.4691227140206733, 'pattern_counts': {'":': 0, '<': 20, '<?xml version=': 0, '>': 20, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 32, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '160907', 'license': 'MIT', 'n_tokens_mistral': 4318, 'n_tokens_neox': 3689, 'n_words': 882, 'path': 'PlanB/Experiment1/Clean_data_txt_throughput_pi.ipynb', 'repo_name': 'phoophoo187/Privacy_SDN_Edge_IoT', 'size': '9315'}