<jupyter_start><jupyter_text>Clean Data of the results of experiment 2- This program is used to export the txt file to dataframe in python. The txt is captured from the experiment2 in Github. File location : https://github.com/tawanmeow/privacy-sdn-logfile/tree/main/Experiment2- Find maximum throughput by sending packets from Edge4 to Edge1 and Edge5 simultaneously. Edge4 perspective (Client)<jupyter_code># import library
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import requests
import io<jupyter_output><empty_output><jupyter_text>Input filename that you want to clean data<jupyter_code># this pandas version dose not support directlly loading the file from url.
url = 'https://github.com/tawanmeow/privacy-sdn-logfile/tree/main/Experiment2'
filename = "/Users/kalika/PycharmProjects/Privacy_SDN_Edge_IoT/PlanB/Experiment2/test2_client.txt"
filename_csv = "/Users/kalika/PycharmProjects/Privacy_SDN_Edge_IoT/PlanB/Experiment2/test2_client.txt"<jupyter_output><empty_output><jupyter_text>Functions for getting the throughput data from dataframe df<jupyter_code>def getDataframeThru(df,start_row,measurement_interval,header_range):
    '''
    This functions will import the data from txt file and return the dataframe without the header of txt file.
    Input: 
        measurement_interval = 30 (sec) : 
        header_range = 10 lines
        start_row = 0 
    Return: 
        df1t : dataframe of througput and jitter
    '''
    
    df1 = df.drop(labels=range(start_row, header_range), axis=0)
    df1t = df1.drop(labels=range(measurement_interval, len(df)), axis=0)
    
    return df1t
def getDatafromTxT(filename, headerrange):
    """
    Get dataframe from txt file:
    filename : xxx.txt
    headerrange : number of lines that needed to be removed. 
    
    return : df : datafame type
    """
    h = headerrange + 1
    skip_1 = list(range(0,h, 1))
    df = pd.read_csv(filename,
        skiprows=skip_1,
        header=None,
        delimiter=' ',
        skipinitialspace=True,
        error_bad_lines=False)
    return df<jupyter_output><empty_output><jupyter_text>Run functions:<jupyter_code>headerrange = 14
df = getDatafromTxT(filename, headerrange)
print(df.head)
## Find start row index of itteration
def getStartEndID(df,start_data,end_data):
    """
    to clean dataframe and return the data with new header
    Input: 
        df : datafram without header of txt file 
    Output
        strat_indices_list : start indices list
    """
    # creating and passing series to new column
    df["Start"]= df[3].str.find(start_data)
    df["End"]= df[3].str.find(end_data)

    index = df.index
    strat_indices = index[df["Start"]==0.0]
    strat_indices_list = strat_indices.tolist()
    end_indices = index[df["End"]==0.0]
    end_indices_list = end_indices.tolist()
    
    
    return strat_indices_list, end_indices_list
    
def getCleanData(df,strat_indices_list,end_indices_list):
    """
    
    """
    df_all = df.drop(labels=range(1, len(df)), axis=0) # create new df
    start_row = 0
    c = 0
    for i in strat_indices_list: 
        
        h = i
        print('h =',h)
        m = end_indices_list[c]
        print('m =', m)
        df1 = getDataframeThru(df,start_row,m,h)
        print('df1 = ', df1)
        result = pd.concat([df_all,df1])
        df_all = result
        c = c + 1
        if i == 0:
            
            df_all = df_all.drop(labels=0, axis=0)
        
        
    return df_all<jupyter_output><empty_output><jupyter_text>Run functions<jupyter_code>start_data = '1.00-2.00'
end_data = '27.00-28.00'
strat_indices_list, end_indices_list = getStartEndID(df,start_data,end_data)
print('strat_indices_list: ', strat_indices_list)
print('end_indices_list: ', end_indices_list)

df_all = getCleanData(df,strat_indices_list,end_indices_list)
# Replace new columns header

df_all_new = df_all.drop(df_all.columns[[1,2,4,6,8,10]], axis=1)  ## remove unnecessary columns
df_all_new.rename({0 : 'Edge', 3 :'Interval', 5 : 'Transfer', 7 :'Bitrate', 8 :'Jitter', 9 :'Lost/Total Datagrams'}, axis=1, inplace=True)
print(df_all_new)<jupyter_output>Edge     Interval Transfer Bitrate  Lost/Total Datagrams  End
0    edge5:    0.00-1.00     1.81    15.2                1310.0 -1.0
1    edge1:    1.00-2.00     1.75    14.7                1270.0 -1.0
2    edge5:    1.00-2.00     1.74    14.6                1260.0 -1.0
3    edge1:    2.00-3.00     1.85    15.5                1340.0 -1.0
4    edge5:    2.00-3.00     1.85    15.5                1340.0 -1.0
..      ...          ...      ...     ...                   ...  ...
698  edge1:  25.00-26.00     1.84    15.4                1330.0 -1.0
699  edge5:  25.00-26.00     1.85    15.5                1340.0 -1.0
700  edge1:  26.00-27.00     1.88    15.8                1360.0 -1.0
701  edge5:  26.00-27.00     1.86    15.6                1350.0 -1.0
702  edge1:  27.00-28.00     1.85    15.5                1340.0  0.0

[1044 rows x 6 columns]<jupyter_text>Plot dataframe<jupyter_code>print(df_all_new['Bitrate'])

df_all_new['Bitrate'] = df_all_new['Bitrate'].astype(float) 
print(len(df_all_new.index))
time = np.array(range(len(df_all_new.index)))
print(time)

df_all_new['Time'] = time
print(df_all_new.head)
df_all_new['Time'] = df_all_new['Time'].astype(int) 

## Throughput
# avergae throughput 
sumThroughput = df_all_new['Bitrate'].sum()
avgSumThroughput = sumThroughput/len(time)
print('avg Throughput :', avgSumThroughput)
print('var Throughput :', df_all_new['Bitrate'].var())
# Throughput 
ax = plt.gca()
df_all_new.plot(kind='line', x='Time', y='Bitrate', ax =ax)
plt.legend(["Throughput"]);
ax.set_xlabel("Time (sec)")
ax.set_ylabel("Throughput (Mbits/s)")
avg_thr = np.ones(len(df_all_new.index),dtype = float)*avgSumThroughput
s = pd.DataFrame(avg_thr, columns = ['AVG Throghput'])
s.plot(ax=ax)
plt.show()<jupyter_output><empty_output><jupyter_text>Jitter<jupyter_code>df_all_new['Jitter'] = df_all_new['Jitter'].astype(float) 
# avergae jitter 
sumThroughput = df_all_new['Jitter'].sum()
avgSumThroughput = sumThroughput/len(time)
print('avg Jitter :', avgSumThroughput)
print('var Jitter :', df_all_new['Jitter'].var())
avg_thr = np.ones(len(df_all_new.index),dtype = float)*avgSumThroughput
J = pd.DataFrame(avg_thr, columns = ['AVG Jitter'])

ax2 = plt.gca()
df_all_new.plot(kind='line', x='Time', y='Jitter', ax =ax2)
plt.legend(["Jitter"]);
ax2.set_xlabel("Time (sec)")
ax2.set_ylabel("Jitter (sec)")
J.plot(ax=ax2)
plt.show()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1269743904309155, 'fraction_numerical': 0.04876552675969943, 'mean_word_length': 3.2323166774821543, 'pattern_counts': {'":': 0, '<': 28, '<?xml version=': 0, '>': 28, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '7', 'dir': 'jupyter-structured-clean-dedup', 'id': '518007', 'license': 'MIT', 'n_tokens_mistral': 2537, 'n_tokens_neox': 2391, 'n_words': 492, 'path': 'PlanB/Experiment2/.ipynb_checkpoints/Clean_data_ex2-checkpoint.ipynb', 'repo_name': 'phoophoo187/Privacy_SDN_Edge_IoT', 'size': '6521'}