<jupyter_start><jupyter_text>** I. Extract Neighbor List per User and per Type **<jupyter_code>!rm -r /nvme/drive_1/NTDS_Final/relations_type_1
!rm -r /nvme/drive_1/NTDS_Final/relations_type_2
!rm -r /nvme/drive_1/NTDS_Final/relations_type_3
!rm -r /nvme/drive_1/NTDS_Final/relations_type_4
!rm -r /nvme/drive_1/NTDS_Final/relations_type_5
!rm -r /nvme/drive_1/NTDS_Final/relations_type_6
!rm -r /nvme/drive_1/NTDS_Final/relations_type_7

!mkdir /nvme/drive_1/NTDS_Final/relations_type_1
!mkdir /nvme/drive_1/NTDS_Final/relations_type_2
!mkdir /nvme/drive_1/NTDS_Final/relations_type_3
!mkdir /nvme/drive_1/NTDS_Final/relations_type_4
!mkdir /nvme/drive_1/NTDS_Final/relations_type_5
!mkdir /nvme/drive_1/NTDS_Final/relations_type_6
!mkdir /nvme/drive_1/NTDS_Final/relations_type_7 
# Seperate the data according to relation type 
with open(data_folder+"relations.csv", "rt") as f_in:
    reader             = csv.reader(f_in, delimiter="\t")
    for i, line in enumerate(reader):
        node_1        = line[2]  
        node_2        = line[3]
        relation_type = int(line[4])
        for node in (node_1,node_2):
            
            f_out = open(data_folder+"relations_type_"+str(relation_type) +\
                        "/"+node+".csv", "at") 
            writer = csv.writer(f_out, delimiter='\t')
            writer.writerow(line)
            f_out.close()
               
        if i%10000 is 0: 
            print("\r"+str(int(i/1000)) +" thousand relations processed", sep=' ', end='', flush=True)<jupyter_output>858240 thousand relations processed<jupyter_text>** II. Extract Local Topology From Neighbor Lists**<jupyter_code>!rm -r /nvme/drive_1/NTDS_Final/type_1
!rm -r /nvme/drive_1/NTDS_Final/type_2
!rm -r /nvme/drive_1/NTDS_Final/type_3
!rm -r /nvme/drive_1/NTDS_Final/type_4
!rm -r /nvme/drive_1/NTDS_Final/type_5
!rm -r /nvme/drive_1/NTDS_Finallocal_list/type_6
!rm -r /nvme/drive_1/NTDS_Finallocal_list/type_7

!mkdir /nvme/drive_1/NTDS_Final/local_list/type_1
!mkdir /nvme/drive_1/NTDS_Final/local_list/type_2
!mkdir /nvme/drive_1/NTDS_Final/local_list/type_3
!mkdir /nvme/drive_1/NTDS_Final/local_list/type_4
!mkdir /nvme/drive_1/NTDS_Final/local_list/type_5
!mkdir /nvme/drive_1/NTDS_Final/local_list/type_6
!mkdir /nvme/drive_1/NTDS_Final/local_list/type_7
# Returns the relations between nodes within num_hops
def local_dfs(num_hops,id_number,relation_type,visited_relations,visited_neighs):
    
    visited_neighs.append(id_number)
    
    if num_hops != 0: 
        local_relations_file = data_folder+"relations_type_"+str(relation_type) +\
                                "/"+str(id_number)+".csv"
        local_relations      = pd.read_csv(local_relations_file,delimiter='\t',
                                    header=None).values 
        for relation in local_relations: 
            current_relation_known = False
            for known_relation in visited_relations:
                if ((relation[0] is known_relation[0]) and \
                    (relation[1] is known_relation[1]) and \
                    (relation[2] is known_relation[2]) and \
                    (relation[3] is known_relation[3]) and \
                    (relation[4] is known_relation[4])): 
                    current_relation_known = True 
                    break
            if not current_relation_known: 
                visited_relation.append(relation)
                
        neighs_1 = local_relations[np.where(local_relations[2]==id_number)][0][:,3]
        neighs_2 = local_relations[np.where(local_relations[3]==id_number)][0][:,2]
        for neigh in np.union1d(neighs_1,neighs_2): 
            if neigh not in visited_neighs: 
                visited_relations, visited_neighs = 
                    local_dfs(num_hops-1,neigh,relation_type,current_relations,visited_neighs)
    return visited_relations,visted_neighs

# Create Local Relation Lists 
for relation_type in [1,2,3,4,5,6,7]: 
    for user in users:
        id_number                     = user[0]
        local_relations, local_neighs = local_dfs(hops,id_number,
                                                  relation_type,list([]),list([]))
        np.savetxt(data_folder+"local_list/type_"+str(relation_type)+\
                   "/"+id_number+".csv",np.asarray(local_relations))
!rm -r data/local_adj/type_1
!rm -r data/local_adj/type_2
!rm -r data/local_adj/type_3
!rm -r data/local_adj/type_4
!rm -r data/local_adj/type_5
!rm -r data/local_adj/type_6
!rm -r data/local_adj/type_7

!mkdir data/local_adj/type_1
!mkdir data/local_adj/type_2
!mkdir data/local_adj/type_3
!mkdir data/local_adj/type_4
!mkdir data/local_adj/type_5
!mkdir data/local_adj/type_6
!mkdir data/local_adj/type_7
for relation_type in [1,2,3,4,5,6,7]: 
    for user in users:
        id_number = user[0]
        relation_list = np.loadtxt(data_folder+"local_list/type_"+str(relation_type)+\
                                   "/"+id_number+".csv",np.asarray(local_relations))<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11382765531062124, 'fraction_numerical': 0.026452905811623247, 'mean_word_length': 2.848111025443331, 'pattern_counts': {'":': 0, '<': 8, '<?xml version=': 0, '>': 8, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '660397', 'license': 'MIT', 'n_tokens_mistral': 1887, 'n_tokens_neox': 1773, 'n_words': 281, 'path': 'raphael_temp/Data Parsing & Filtering.ipynb', 'repo_name': 'ilijagjorgjiev/ntds2018_team50', 'size': '4990'}