<jupyter_start><jupyter_text>DGL Graph<jupyter_code>data_dict = dict()
data_dict[('Computer', 'Connect', 'Computer')] = list(comp_comp_dict)
data_dict[('User', 'Logon', 'Computer')] = list(user_comp_dict)
data_dict[('User', 'SwitchUser', 'User')] = list(user_user_dict)
data_dict[('Computer', 'Create', 'Process')] = list(comp_proc_dict)
g = dgl.heterograph(data_dict)
g<jupyter_output><empty_output><jupyter_text>Pytorch_geometric graph<jupyter_code>data_dict[('Computer', 'Connect', 'Computer')] = torch.transpose(torch.LongTensor(data_dict[('Computer', 'Connect', 'Computer')]), 0, 1) 

data_dict[('User', 'Logon', 'Computer')] = torch.transpose(torch.LongTensor(data_dict[('User', 'Logon', 'Computer')]), 0, 1) 

data_dict[('User', 'SwitchUser', 'User')] = torch.transpose(torch.LongTensor(data_dict[('User', 'SwitchUser', 'User')]), 0, 1) 

data_dict[('Computer', 'Create', 'Process')] = torch.transpose(torch.LongTensor(data_dict[('Computer', 'Create', 'Process')]), 0, 1) 
data_dict[('User', 'SwitchUser', 'User')]
x = torch.arange(14).view(2, 7)
torch.flip(x, [0, 1])
data_dict[('Computer', 'Connect', 'Computer')]
from torch_geometric.data import HeteroData

data = HeteroData()
data['Computer', 'Connect', 'Computer'].edge_index = data_dict[('Computer', 'Connect', 'Computer')]
data['User', 'Logon', 'Computer'].edge_index = data_dict[('User', 'Logon', 'Computer')]
data['User', 'SwitchUser', 'User'].edge_index = data_dict[('User', 'SwitchUser', 'User')]
data['Computer', 'Create', 'Process'].edge_index = data_dict[('Computer', 'Create', 'Process')]
from csv import reader
# open file in read mode
count = 0
day_in_second = 60*60*24
day_filter = [2, 3, 6, 7, 8, 9, 10, 13, 14, 15, 16, 21, 22, 23, 27, 28, 29, 30] # only get data from this list of day
user_comp_dict = set()
comp_comp_dict = set()
user_user_dict = set()
with open('/home/andrewngo/Desktop/LANL_data/auth.txt', 'r') as read_obj:
    # pass the file object to reader() to get the reader object
    csv_reader = reader(read_obj)
    # Iterate over each row in the csv using reader object
    for row in csv_reader:
        if auth_filter(row, day_filter):
            continue
        if np.ceil(int(row[0]) / day_in_second) > 30:
            break
        count = count + 1
        if count % 1000000 == 0:
            print(str(count/1000000) + "Mth lines")
        source_user     = row[1].split("@")[0]
        des_user        = row[2].split("@")[0]
        source_computer = row[3]
        des_computer    = row[4]
        if source_user not in user2nodeid:
            user2nodeid[source_user] = len(user2nodeid) 
        if des_user not in user2nodeid:
            user2nodeid[des_user] = len(user2nodeid) 
        if source_computer not in computer2nodeid:
            computer2nodeid[source_computer] = len(computer2nodeid) 
        if des_computer not in computer2nodeid:
            computer2nodeid[des_computer] = len(computer2nodeid)
# Comp -> comp edge
        if source_computer != des_computer:
            comp_comp_dict.add((computer2nodeid[source_computer], computer2nodeid[des_computer]))
# user -> comp edge
        user_comp_dict.add((computer2nodeid[source_computer], user2nodeid[source_user]))
        user_comp_dict.add((computer2nodeid[des_computer], user2nodeid[des_user]))
# user -> user edge
        if source_user != des_user:
            user_user_dict.add((user2nodeid[source_user], user2nodeid[des_user]))
        temp = (row[0], source_user, row[3], row[4])
        if temp in df_red_tuple:
            df_red_tuple[temp] = df_red_tuple[temp] + 1
            
# df = pd.DataFrame (df)
# df 
df_red_tuple
df=pd.read_csv('/home/andrewngo/Desktop/LANL_data/auth_1000000.txt', sep =',' , header=None)
df
df=pd.read_csv('/home/andrewngo/Desktop/LANL_data/auth_1000000.txt', sep =',' , header=None)
df_red = pd.read_csv('/home/andrewngo/Desktop/LANL_data/redteam.txt', sep =',' , header=None)
df_proc=pd.read_csv('/home/andrewngo/Desktop/LANL_data/proc_1000000.txt', sep =',' , header=None)
df_red
# df_red_tuple = list(df_red.to_records())
df_red[1], _= zip(*df_red[1].str.split('@').tolist())
df_red_tuple = list(df_red.itertuples(index=False, name=None))
df_red_tuple = {i : 0 for i in df_red_tuple}
df_red_tuple
len(df_red_tuple)
device = 'cuda' if torch.cuda.is_available() else 'cpu'
temp = [(9605, 11336, 6490), (9605, 11336, 4702), (9605, 11336, 10700), (9605, 11336, 6270)]
temp = [torch.LongTensor(i).to(device) for i in temp]
temp
df = df.sort_values(by=[0]).reset_index(drop=True)
df
# normal 
# malicious (Lateral movement activites)



# unsupervised learnign 

# training : normal 
# validation & testing: normal adn malicious 

    
    

# graph -> metapath2vec -> path embedding  -> autoencoder -> malicious/normal  
    
    
len(set(df[1]))
set(df[7])
set(df[6])
len(set(df_red[2]))
df_proc = df_proc.sort_values([0]).reset_index(drop=True)
df_proc
df_proc.groupby([2,4]).count()
df_red
len(set(df_red[0]))
df_red = df_red.sort_values([0])
df_red
len(set(df_red[2]))
len(set(df_red[1]))
len(set(df_red[2]))
len(set(df_red[3]))
df[8].value_counts()
7626505167/59<jupyter_output><empty_output><jupyter_text>Command to split txt into small files  Red team label  auth.txtx file: “time, source user@domain, destination user@domain, source computer, destination computer, authentication type, logon type, authentication orientation, success/failure” + Source User-> Source Computer: Logon+ Source Computer -> Destination Computer: Connection+ Source User -> Process: Create+ Source Computer -> Process: CreateRedteam file: “time,user@domain,source computer,destination computer”proc.txt: “time,user@domain,computer,process name,start/end”<jupyter_code>day_in_second = 60*60*24
day_in_second
# get dictionary of day that involved malicious activities
df_red["time_day"] = df_red[0] / day_in_second
df_red["time_day_round"] = np.ceil(df_red[0] / day_in_second)
df_red
import matplotlib.pyplot as plt
import numpy as np


# max length path
max_length = 40
# path_from_user
# x = np.zeros(shape=max_length)
x = list(df_red["time_day_round"])
# plot histogram  
labels, counts = np.unique(x, return_counts=True)
plt.bar(labels, counts, align='center')
plt.gca().set_xticks(labels)
plt.show()
plt.show() 
from csv import reader
# open file in read mode
day_filter = [2, 3, 6, 7, 8, 9, 10, 13, 14, 15, 16, 21, 22, 23, 27, 28, 29, 30] # only get data from this list of day
with open('/home/andrewngo/Desktop/LANL_data/auth_1000000.txt', 'r') as read_obj:
    # pass the file object to reader() to get the reader object
    csv_reader = reader(read_obj)
    # Iterate over each row in the csv using reader object
    for row in csv_reader:
        count = 1
temp = "U9862@DOM1"
temp.split("@")[0]
len(user)
len(comp_comp_dict)
len(comp_user_dict)
len(user_user_dict)
len(set(comp_proc_dict))
len(computer2nodeid)
len(process2nodeid)
df_proc
np.ceil(1185549 / day_in_second)
labels
# Create set of tuple that indicate the malicious log
red_set = []
for idx_red, row_red in df_red.iterrows():
    red_set.append(tuple(row_red))

red_set = set(red_set)
red_set
# Add is_malicious columns on the original df
df_compare = df[[0, 1, 3, 4]]
df_compare_dict = df_compare.to_dict('records')
is_malicious = []
for row in tqdm(df_compare_dict):
    row = (row[0], row[1], row[3], row[4])
    if tuple(row) in red_set:
        print("is_malicious")
        is_malicious.append(1)
    else:
        is_malicious.append(0)
        
df['is_malicious'] = is_malicious
df["source_user"], df["source_domain"] = zip(*df[1].str.split('@').tolist())
df["source_user"]=df["source_user"].str.rstrip('$')
df["destination_user"], df["destination_domain"] = zip(*df[2].str.split('@').tolist())
df["destination_user"]=df["destination_user"].str.rstrip('$')
df['same_user']=(df['destination_user']==df['source_user'])
df['same_domain']=(df['destination_domain']==df['source_domain'])

df['source_user_comp_same']=(df[3]==df['source_user'])
df['destination_user_comp_same']=(df['destination_user']==df[4])
df['same_comp']=(df[3]==df[4])
df['source_domain_comp_same']=(df[3]==df['source_domain'])
df['destination_domain_comp_same']=(df['destination_domain']==df[4])
df[5].loc[df[5].str.contains("MICROSOFT")] = "MICROSOFT_AUTHENTICATION_PACKET_V1_0"
df["source_start_with_U"] = df.source_user.str.startswith('U')
df["destination_start_with_U"] = df.destination_user.str.startswith('U')
df["source_start_with_C"] = df.source_user.str.startswith('C')
df["destination_start_with_C"] = df.destination_user.str.startswith('C')
df[df["source_start_with_U"]].reset_index(drop=False)
count = 0
for i in df["destination_start_with_U"]:
    if i == True:
        count = count + 1
count<jupyter_output><empty_output><jupyter_text>Graph Contruction<jupyter_code># construct LOGON graph based on "?" authentication
# Step 1: construct nodeid dictionary for user, process and computer (for easier look up)
df = df[df[8] == "Success"] # df with success authentication only
df
# 

# there is some duplicated edge in the graph (e.g a user could logon to a computer many time)
# we temporary ignore these duplicated edge, i.e ignore the temporal information of the graph
reduced_edge = True 
df.groupby([5,6,7,"source_start_with_U"]).count()
# generate user to nodeid lookup dictionary
set_user = set(df[~df["source_start_with_C"]]["source_user"])
set_user.update(list(set(df[~df["destination_start_with_C"]]["destination_user"])))
set_user = list(set_user)
user2nodeid = {set_user[i]: i for i in range(len(set_user))}
len(user2nodeid)
# generate computer to nodeid lookup dictionary
set_computer = set(df[3])
set_computer.update(list(set(df[4])))
set_computer = list(set_computer)
computer2nodeid = {set_computer[i]: i for i in range(len(set_computer))}
len(computer2nodeid)
set_proc = list(set(df_proc[3]))
# set_computer = list(set_computer)
proc2nodeid = {set_proc[i]: i for i in range(len(set_proc))}
proc2nodeid
# Step 2: contruct comp -> comp using same_comp
mask_comp_comp = ~df["same_comp"]
if reduced_edge == True:
    comp_source = list(set(map(computer2nodeid.get,df[mask_comp_comp][3]))) # convert computer to nodeid
    comp_des = list(set(map(computer2nodeid.get,df[mask_comp_comp][4]))) # convert computer to nodeid
else:
    comp_source = list(map(computer2nodeid.get,df[mask_comp_comp][3])) # convert computer to nodeid
    comp_des = list(map(computer2nodeid.get,df[mask_comp_comp][4])) # convert computer to nodeid    

comp_comp_dict = list(zip(comp_source, comp_des))
comp_comp_dict
# Step 3: contruct user --> comp 
# in this version, we only consider "real" user: in this data, there is the presense of 2 type of user, user start
# with character "U" (U66@DOM1) and with "C" (C436$@DOM1), we think that the C is represent for a computer
# case 1: source and destination user, source and destination computer are the same
mask_user_comp = (~(df["source_start_with_C"] | df["destination_start_with_C"]) & (df["same_comp"] & df["same_user"]))


if reduced_edge == True: 
    user_source = list(set(map(user2nodeid.get,df[mask_user_comp]["source_user"]))) # convert computer to nodeid
    comp_source = list(set(map(computer2nodeid.get,df[mask_user_comp][3]))) # convert computer to nodeid
else:
    user_source = list(map(user2nodeid.get,df[mask_user_comp]["source_user"])) # convert computer to nodeid
    comp_source = list(map(computer2nodeid.get,df[mask_user_comp][3])) # convert computer to nodeid
user_comp_dict_case_1 = list(zip(user_source, comp_source))
print(len(user_comp_dict_case_1))
user_comp_dict = user_comp_dict_case_1


# Case 2: other case
# destination -> comp as 2 difference edge
mask_user_comp = (~(df["source_start_with_C"] | df["destination_start_with_C"]) & ~(df["same_comp"] & df["same_user"]))

if reduced_edge == True: 
    user_source = list(set(map(user2nodeid.get,df[mask_user_comp]["source_user"]))) # convert computer to nodeid
    comp_source = list(set(map(computer2nodeid.get,df[mask_user_comp][3]))) # convert computer to nodeid
    
    user_des = list(set(map(user2nodeid.get,df[mask_user_comp]["destination_user"]))) # convert computer to nodeid
    comp_des = list(set(map(computer2nodeid.get,df[mask_user_comp][4]))) # convert computer to nodeid
else: 
    user_source = list(map(user2nodeid.get,df[mask_user_comp]["source_user"])) # convert computer to nodeid
    comp_source = list(map(computer2nodeid.get,df[mask_user_comp][3])) # convert computer to nodeid
    
    user_des = list(set(user2nodeid.get,df[mask_user_comp]["destination_user"])) # convert computer to nodeid
    comp_des = list(set(computer2nodeid.get,df[mask_user_comp][4])) # convert computer to nodeid  
user_comp_dict_case_2 = list(zip(user_source, comp_source))
user_comp_dict_case_2.extend(list(zip(user_des, comp_des)))

print(len(user_comp_dict_case_2))
user_comp_dict.extend(user_comp_dict_case_2)

user_comp_dict


# Step 4: User -> User: user switch
mask_user_user = (~(df["source_start_with_C"] | df["destination_start_with_C"]) & ~(df["same_user"]))

if reduced_edge == True: 
    user_source = list(set(map(user2nodeid.get,df[mask_user_comp]["source_user"]))) # convert computer to nodeid
    user_des = list(set(map(user2nodeid.get,df[mask_user_comp]["destination_user"]))) # convert computer to nodeid
else:
    user_source = list(map(user2nodeid.get,df[mask_user_comp]["source_user"])) # convert computer to nodeid
    user_des = list(map(user2nodeid.get,df[mask_user_comp]["destination_user"])) # convert computer to nodeid
    
user_user_dict = list(zip(user_source, user_des))
print(len(user_user_dict))
# Step 5: construct comp -> process
df_proc
if reduced_edge == True:
    comp = list(set(map(computer2nodeid.get,df_proc[2]))) # convert computer to nodeid
    proc = list(set(map(proc2nodeid.get,df_proc[3]))) # convert computer to nodeid
else:
    comp = list(map(computer2nodeid.get,df_proc[2])) # convert computer to nodeid
    proc = list(map(proc2nodeid.get,df_proc[3])) # convert computer to nodeid

comp_proc_dict = list(zip(comp, proc))
print(len(comp_proc_dict))
# Step 6: generate graph
data_dict = dict()
data_dict[('Computer', 'Connect', 'Computer')] = comp_comp_dict
data_dict[('User', 'Logon', 'Computer')] = user_comp_dict
data_dict[('User', 'SwitchUser', 'User')] = user_user_dict
data_dict[('Computer', 'Create', 'Process')] = comp_proc_dict

g = dgl.heterograph(data_dict)
g<jupyter_output><empty_output><jupyter_text>Conclusion In this dataset, Fails take roughly 1%. If it is a representative sample of the real data, then running machine learning on the whole set will just not make sense. Any classifier that just predict "Success" for every line will attain 99% accuracy. This means that I need to collect data for "Fail" cases and randomly sample data for "Success" in roughly equal amounts and then look at machine learning (classifier) for such sets to see the true accuracy of the algorithm.  Next Steps I want to examine this dataset to see if there are any obvious correlations and to understand the data I have in my columns.  authentication type<jupyter_code>df[5].unique()<jupyter_output><empty_output><jupyter_text>logon type<jupyter_code>mask = (df[7] == 'LogOn')
# mask = (df[6] == 'TGS')
pos = np.flatnonzero(mask)
df.iloc[pos]
df[6].unique()<jupyter_output><empty_output><jupyter_text>authentication orientation<jupyter_code>df[7].unique()
df_suc.groupby([5,6,7]).count()
df.groupby([7,8]).count()
df.groupby([7,8]).count()
df.groupby([6,7]).count()<jupyter_output><empty_output><jupyter_text>This is a simple way to see if there are any labels in columns 5-7 that predict the outcome. (answer: not really as the count for most events that can be interpreted this way is too low). Also I am try to see if there are any interesting correlations between labels.<jupyter_code>print(len(df[3].unique()), len(df[4].unique()))<jupyter_output>13796 12411<jupyter_text>Potentially too many variables to be used in analysis<jupyter_code>df['same_user'].value_counts()
df['same_domain'].value_counts()
df['source_domain'].unique()
df['destination_domain'].unique()
df['source_user'].unique()
df['destination_user'].unique()<jupyter_output><empty_output><jupyter_text>Potentially too many variable. I now want to explore what users I have in addition to C-numbers and U-numbers. (C=computer and U=user?)<jupyter_code>good=df[~df.source_user.str.startswith("U")] 
good=good.source_user[~good.source_user.str.startswith('C')]
good.unique()
good=df[~np.logical_or(df.destination_user.str.startswith("U"), df.destination_user.str.startswith("C"))] 
#good=good.destination_user[~good.destination_user.str.contains('C')]
good.destination_user.unique()<jupyter_output><empty_output><jupyter_text>Idea: one can expand this column into into 6 categories: C-users, U-users, 'ANONYMOUS LOGON', 'LOCAL_SERVICE', 'SYSTEM', 'NETWORK SERVICE'<jupyter_code>dd=df['destination_domain'].str.startswith('C')
print(min(df['destination_domain'][dd].str.slice(1).astype(int)), max(df['destination_domain'][dd].str.slice(1).astype(int)))
dd = df[~df.destination_domain.str.startswith('C')]
print(dd.destination_domain.unique())
sd=df['source_domain'].str.startswith('C')
print(min(df['source_domain'][sd].str.slice(1).astype(int)), max(df['source_domain'][sd].str.slice(1).astype(int)))
sd=df[~df.source_domain.str.startswith('C')]
print(sd.source_domain.unique())
df_proc<jupyter_output><empty_output><jupyter_text>Conclusion This dataset contains columns of categorical data (aside from time). To work with this data, each label should be converted to its own column with values 1 (True) if the label applies and 0 (False) otherwise. Some columns (5-7) contain ~10 labels, where as other columns contain ten thousands of labels. I will ignore the 2nd class of labels on the first pass. Instead I will consider when these labels coincide. This way I will prevent my set of features from exploding. Also the 2nd class of labels most likely comes from some ordering of computers and users in the lab. Considering if one wants to authenticate to the same computer or to a different computer should matter more for authentication success than specific computer label.<jupyter_code>df
df['source_user_comp_same'].value_counts()
df['destination_user_comp_same'].value_counts()
df['same_comp'].value_counts()
df['source_domain_comp_same'].value_counts()
df['destination_domain_comp_same'].value_counts()
data = torch.load('/home/andrewngo/Desktop/MLTracker/graph_data.pt')
data
data['Computer'].num_nodes
import os.path as osp
from xxlimited import new
import os, datetime
import torch
import time
import os, datetime
from torch_geometric.datasets import AMiner
path = osp.join(osp.dirname(osp.realpath(__file__)), '../../data/AMiner')
dataset = AMiner(path)
normal_path = torch.load("/home/andrewngo/Desktop/MLTracker/graph_data_20220219110042/normal_path.pt")
malicious_train_val_path = torch.load("/home/andrewngo/Desktop/MLTracker/graph_data_20220219110042/malicious_train_val_path.pt")
malicious_test_path = torch.load("/home/andrewngo/Desktop/MLTracker/graph_data_20220219110042/malicious_test_path.pt")
normal_path
malicious_train_val_path
malicious_test_path
torch.load("/home/andrewngo/Desktop/MLTracker/model_20220219153022/out_mal_test.pt")<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.14143571577411962, 'fraction_numerical': 0.024327984996874347, 'mean_word_length': 6.16840926064227, 'pattern_counts': {'":': 0, '<': 48, '<?xml version=': 0, '>': 65, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '12', 'dir': 'jupyter-structured-clean-dedup', 'id': '123612', 'license': 'MIT', 'n_tokens_mistral': 7007, 'n_tokens_neox': 6632, 'n_words': 1492, 'path': 'exploration.ipynb', 'repo_name': 'tingkelvin/LMTracker', 'size': '19196'}