<jupyter_start><jupyter_text>Dataset Information<jupyter_code>df.sample(10)
df.info()
df.isna().sum()
df['category'] = df['category']+'-'+df['subcategory']
df.groupby('category').agg({'category':'first', 'category':'count'}).rename_axis(None).sort_values('category', ascending=False)<jupyter_output><empty_output><jupyter_text>Dataset Preprocecing<jupyter_code>a = df.loc[df['category'] == 'DoS-UDP']
b = df.loc[df['category'] == 'DDoS-TCP']
c = df.loc[df['category'] == 'DDoS-UDP']
d = df.loc[df['category'] == 'DoS-TCP']
e = df.loc[df['category'] == 'DDoS-HTTP']
f = df.loc[df['category'] == 'DoS-HTTP']
g = df.loc[df['category'] == 'Normal-Normal']
newData = pd.concat([a,b,c,d,e,f,g], axis=0, join='outer', ignore_index=False)
newData['category'] = newData['category'].replace(['DoS-TCP'], 'DoS-TCP')
newData['category'] = newData['category'].replace(['DoS-UDP'], 'DoS-UDP')
newData['category'] = newData['category'].replace(['DDoS-TCP'], 'DDoS-TCP')
newData['category'] = newData['category'].replace(['DDoS-UDP'], 'DDoS-UDP')
newData['category'] = newData['category'].replace(['DDoS-HTTP'], 'DDoS-HTTP')
newData['category'] = newData['category'].replace(['DoS-HTTP'], 'DoS-HTTP')
# newData['category'] = newData['category'].replace(['Reconnaissance-Service_Scan-Service_Scan'], 'Reconnaissance-Service-Scan')
newData['category'] = newData['category'].replace(['Normal-Normal'], 'Normal')
newData.drop([
  'pkSeqID',
  'stime',
  'flgs',
  'proto',
  'saddr',
  'sport',
  'daddr',
  'dport',
  'state',
  'seq',
  'flgs_number',
  'ltime',
  'attack',
  'stddev',
  'sum',
  'mean',
  'min',
  'max',
  'proto_number',
  'state_number',
  'TnBPSrcIP',
  'TnBPDstIP',
  'TnP_PSrcIP',
  'TnP_PDstIP',
  'TnP_PerProto',
  'TnP_Per_Dport',
  'AR_P_Proto_P_SrcIP',
  'AR_P_Proto_P_DstIP',
  'N_IN_Conn_P_DstIP',
  'N_IN_Conn_P_SrcIP',
  'AR_P_Proto_P_Sport',
  'AR_P_Proto_P_Dport',
  'Pkts_P_State_P_Protocol_P_DestIP',
  'Pkts_P_State_P_Protocol_P_SrcIP',
  'subcategory',
], axis=1, inplace=True)
newData.sample(10)
newData.shape
newData.info()
newData.groupby('category').agg({'category':'first', 'category':'count'}).rename_axis(None).sort_values('category', ascending=False)<jupyter_output><empty_output><jupyter_text>Export Data to CSV<jupyter_code>newData.to_csv("UNSW_2018_IoT_Botnet_V6.csv", index=False)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.22525597269624573, 'fraction_numerical': 0.00469283276450512, 'mean_word_length': 7.950381679389313, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '337664', 'license': 'MIT', 'n_tokens_mistral': 984, 'n_tokens_neox': 967, 'n_words': 107, 'path': 'Preprocecing Data.ipynb', 'repo_name': 'skyapps-id/IDS-ML', 'size': '2344'}