<jupyter_start><jupyter_text>Importing necessary libraries<jupyter_code>import pandas as pd
import numpy as np
#creating a dataframe to read the dataset after mounting in google drive
raw_csv_data = pd.read_csv("extracted_attributes-consolidated.csv")
raw_csv_data.info()
#eye balling the data
raw_csv_data.head(5)<jupyter_output><empty_output><jupyter_text>Formatting "Library" column<jupyter_code>raw_csv_data['Library'] =raw_csv_data['Library'].astype(str).str.replace("[","").str.replace("]","")
raw_csv_data['Library']=raw_csv_data['Library'].str.replace('\s*,\s*',',')
raw_csv_data['Library'] = raw_csv_data['Library'].str.upper()
raw_csv_data.head(3)<jupyter_output><empty_output><jupyter_text>Creating dummy variable for "Library"<jupyter_code>#copying the old dataframe with a new dataframe meanwhile to mess around
df = raw_csv_data.copy()
encoded_library=df['Library'].str.get_dummies(sep=',')
df=pd.concat([df,encoded_library], axis=1)
pd.options.display.max_columns = None
pd.options.display.max_rows = None
display(df.head(10))<jupyter_output><empty_output><jupyter_text>**Observation 1:**Created new columns with unique libraries and passing boolean representation if a library call was done for a particular resource  Formatting "Language" column<jupyter_code>df['Language'] = df['Language'].astype(str).str.replace("[","").str.replace("]","")
df['Language'] = df['Language'].str.replace('\s*,\s*',',')
df['Language'] = df['Language'].str.upper()
df.head(3)<jupyter_output><empty_output><jupyter_text>Creating dummy variable for "Language"<jupyter_code>#copying the old dataframe with a new dataframe meanwhile to mess around
df2 = df.copy()
encoded_library2=df2['Language'].str.get_dummies(sep=',')
df2=pd.concat([df2,encoded_library2], axis=1)
pd.options.display.max_columns = None
pd.options.display.max_rows = None
display(df2.head(10))<jupyter_output><empty_output><jupyter_text>**Observation 2:**Created new columns with unique languages and passing boolean representation if a certain language was used for a particular resource  Unique Library calls made and Langauage used<jupyter_code>preprocessed_data = df2.copy()
preprocessed_data.head(3)
preprocessed_data = preprocessed_data.drop('Language', 1)
preprocessed_data = preprocessed_data.drop('Library', 1)



preprocessed_data = preprocessed_data.rename(columns={"APTGroup": "APT Group"})
preprocessed_data.head(3)
from sklearn.preprocessing import LabelEncoder

preprocessed_data['APTGroup'] = LabelEncoder().fit_transform(preprocessed_data['APT Group'])
preprocessed_data['APTGroup'].max()
preprocessed_data.head(5)

#preprocessed_data.dropna(axis=0, how='any', inplace=True)
preprocessed_data.isnull().sum()
#preprocessed_data.info()

#preprocessed_data.tail(5)
preprocessed_data.to_csv('final_preprocessed_dataset.csv', index=True)
%ls<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1416022487702038, 'fraction_numerical': 0.009135628952916374, 'mean_word_length': 9.703007518796992, 'pattern_counts': {'":': 1, '<': 25, '<?xml version=': 0, '>': 25, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '468048', 'license': 'MIT', 'n_tokens_mistral': 975, 'n_tokens_neox': 937, 'n_words': 174, 'path': '2. Preprocessing/Notebook/Preprocessing.ipynb', 'repo_name': 'PritomDas/Cyber-Attack-Attribution-with-Machine-Learning', 'size': '2846'}