<jupyter_start><jupyter_text>This script is for our second visulization<jupyter_code>import pandas as pd
import json
with open('../data/json-converted/assignment-1/1-tsv-to-json.json') as f:
    j = json.load(f)
df = pd.DataFrame(j['Email'])<jupyter_output><empty_output><jupyter_text>We are going to see the popularity of each attack type<jupyter_code>df2 = df[['reconnaissance','social_engineering','malware','credential_phishing']]<jupyter_output><empty_output><jupyter_text>We need the format of  index   genre  1          a,b,c  2           b,d<jupyter_code>list = []
for index, rows in df2.iterrows():
    temp = []
    if float(rows['reconnaissance']) == 1.0:
        temp.append('RE')
    if float(rows['social_engineering']) == 1.0:
        temp.append('SE')
    if float(rows['malware']) == 1.0:
        temp.append('MA')
    if float(rows['credential_phishing']) == 1.0:
        temp.append('CP')
#     Get the keywords we want

    if len(temp) == 1:
        text = temp[0]
        list.append(text)
    elif len(temp) > 1:
#     We are eliminating the rows with 4 '0's. Which means there are no attack types categorized for that row.
        text = ''
        for i in temp:
            text = text + ',' + i
        text = text.lstrip(',')
        list.append(text)
df_new = pd.DataFrame(list,columns=['genre'])
df_new.to_csv("../data/visualizations/vis2/vis2data.csv")<jupyter_output><empty_output><jupyter_text>d3 observable url  https://observablehq.com/@drizzle98/attack-type-venn<jupyter_code>df_new<jupyter_output><empty_output><jupyter_text>Data processing for task5<jupyter_code>index = 0
json_string = ''
for m,n in df_new.iterrows():
    new = {}
    new2 = {}
    temp = {}
    temp['_index'] = 'attack_type'
    temp['_id'] = index
    new['index'] = temp
    
    with open("../../DSCI-550-Assignment-3/data/visualizations/vis2data.json", "a") as write_file:
        json.dump(new, write_file)
        write_file.write('\n')
    json_string = json_string + c
    a = n['genre']
    new2['genre'] = a
    index = index + 1
    
    with open("../../DSCI-550-Assignment-3/data/visualizations/vis2data.json", "a") as write_file:
        json.dump(new2, write_file)
        write_file.write('\n')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1529568697198755, 'fraction_numerical': 0.017785682525566917, 'mean_word_length': 3.2938931297709924, 'pattern_counts': {'":': 0, '<': 21, '<?xml version=': 0, '>': 22, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '455156', 'license': 'MIT', 'n_tokens_mistral': 826, 'n_tokens_neox': 801, 'n_words': 157, 'path': 'notebooks/vis2conversion.ipynb', 'repo_name': 'Anthonyive/DSCI-550-Assignment-3', 'size': '2249'}