<jupyter_start><jupyter_text>Read Data<jupyter_code>import numpy as np 
import pandas as pd 
for dirname, _, filenames in os.walk('../archive/'):
    for filename in filenames:
        print(os.path.join(dirname, filename))
df = pd.read_csv("../archive/sqliv2.csv",encoding='utf-16')<jupyter_output><empty_output><jupyter_text>Pre-Process<jupyter_code>df.shape
data = df.drop_duplicates('Sentence')
data.shape
data.to_csv('../data/all.csv', index=None)

attack = data[data['Label']==1][1:]
attack = attack.sample(frac=1).reset_index(drop=True)
attack.shape
normal = data[data['Label']==0]
normal = normal.sample(frac=1).reset_index(drop=True)
normal.shape
r1 = 0.8
r2 = 0.9
a1 = attack[:int(attack.shape[0]*r1)]
a2 = attack[int(attack.shape[0]*r1):int(attack.shape[0]*r2)]
a3 = attack[int(attack.shape[0]*r2):]
a1
a2
a3
normal
n1 = normal[:int(normal.shape[0]*r1)]
n2 = normal[int(normal.shape[0]*r1):int(normal.shape[0]*r2)]
n3 = normal[int(normal.shape[0]*r2):]
n1
train = a1.append(n1)
val = a2.append(n2)
test = a3.append(n3)
train = train.sample(frac=1).reset_index(drop=True)
val = val.sample(frac=1).reset_index(drop=True)
test = test.sample(frac=1).reset_index(drop=True)
train
val
test
train.to_csv('../data/train.csv', index=None)
val.to_csv('../data/val.csv', index=None)
test.to_csv('../data/test.csv', index=None)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.19690265486725664, 'fraction_numerical': 0.036135693215339236, 'mean_word_length': 6.456043956043956, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '589013', 'license': 'MIT', 'n_tokens_mistral': 572, 'n_tokens_neox': 558, 'n_words': 79, 'path': 'performance_test/train/data_process.ipynb', 'repo_name': 'Lionelsy/SQL-Injection-Detection-via-Deep-Learning', 'size': '1356'}