<jupyter_start><jupyter_text>RUN cells below for costed analysis:<jupyter_code>import networkx as nx
import numpy as np
import random
import os
import os
from shutil import copyfile
from tqdm import tqdm

data_name = "HI-II-14" #HI-II-14, Digg
experiment_type = "ND"
step_ratio = 0.01
read_file_inverse=False
torch_costs = ['degree', 'zero', '001']
tf_costs = ['random', 'degree', 'zero', '001']
network_file_name = "{}_degree".format(data_name)
network_file_path = "./data/real/cost/{}.gml".format(network_file_name)
G = nx.read_gml(network_file_path, destringizer=int)
print("Number of nodes in the graph:", G.number_of_nodes())
print("Number of edges in the graph:", G.number_of_edges())

_g_num_nodes = G.number_of_nodes()
_CN_denom = (((_g_num_nodes * (_g_num_nodes - 1))) /2)

print(_g_num_nodes)
def calc_graph_connectivity(G, experiment_type=experiment_type):
    if(G.number_of_nodes() in [0, 1]): return 0
    if(experiment_type=="CN"):
        pairwise_connectivity = 0
        for i in list(nx.connected_components(G)): pairwise_connectivity += (len(i) * (len(i) -1)) / 2
        pc = pairwise_connectivity / _CN_denom
        return pc
    else:
        maxCC = len(max(nx.connected_components(G), key=len))
        #print(maxCC , _g_num_nodes)
        return maxCC / _g_num_nodes

SUM_COST = 0
for i in range(len(G.nodes())):
    SUM_COST += G.nodes[i]['weight']

print(SUM_COST)
# high degree adaptive
def cost_HDA(G, k=k, write=True):
    pairwise_conn_list = []
    cost_list = [0]
    pairwise_conn_list.append(calc_graph_connectivity(G))

    print("HDA for k: {}".format(k))

    while(True):
        print("- ", end="")
        degrees = dict(G.degree)
        degrees_sorted = list(sorted(degrees.items(), key=lambda item: item[1], reverse=True))
        isTerminal = k > len(degrees_sorted)

        num_removals = int(min(k, len(degrees_sorted)))
        node_degree_pairs = degrees_sorted[0:num_removals]
        node_removals = [i[0] for i in node_degree_pairs]

        curr_cost = cost_list[-1]
        for n in node_removals:
            curr_cost += G.nodes[n]['weight']
        
        cost_list.append(curr_cost)

        G.remove_nodes_from(node_removals)

        pairwise_conn_list.append(calc_graph_connectivity(G))

        if(isTerminal): break

    print("\t-Finished calculations")

    if(write):
        result_file = os.path.join(HDA_export_path, "{}_scores_k_{}.txt".format(data_name, k))
        with open(result_file, 'w') as f_out:
                for j in range(len(pairwise_conn_list)):
                    f_out.write('%.8f\n' % pairwise_conn_list[j])
        print("\t-Finished export")

    return pairwise_conn_list, cost_list
def cost_calc_score_from_nodes_file(G, node_files, export_path, k=k, write=True, start_idx=1, read_file_inverse=False):
    pairwise_conn_list = []
    cost_list = [0]
    pairwise_conn_list.append(calc_graph_connectivity(G))

    f = open(node_files,'r')
    node_removals = []
    for idx, line in enumerate(f.readlines()):
        nodes_line = line.split(",")[start_idx:]
        node_removals = node_removals + nodes_line

    
    node_removals = [int(node_removals[i]) for i in range(len(node_removals))]
    if(read_file_inverse):
        node_removals = node_removals[::-1]
    while(True):
        print("- ", end="")

        isTerminal = k > len(node_removals)

        num_removals = int(min(k, len(node_removals)))
        curr_node_removals = node_removals[0:num_removals]
        #print(curr_node_removals)
        #print("=======")

        curr_cost = cost_list[-1]
        for n in curr_node_removals:
            curr_cost += G.nodes[n]['weight']
        
        cost_list.append(curr_cost)

        G.remove_nodes_from(curr_node_removals)

        pairwise_conn_list.append(calc_graph_connectivity(G))

        node_removals = node_removals[num_removals:]
        if(isTerminal): break

    print("\t-Finished calculations")
    if(write):
        result_file = os.path.join(export_path, "scores_k_{}.txt".format(k))
        with open(result_file, 'w') as f_out:
                for j in range(len(pairwise_conn_list)):
                    f_out.write('%.8f\n' % pairwise_conn_list[j])

        print("\t-Finished export")


    return pairwise_conn_list, cost_list
G = nx.read_gml(network_file_path, destringizer=int)
HDA_scores, HDA_costs = cost_HDA(G, k=k, write=False)

for i in range(len(HDA_costs)): HDA_costs[i] =  HDA_costs[i]/SUM_COST
cost_exp_dict = {
                "HDA_robustness_l": [HDA_costs,HDA_scores],
                
                }
for weight_type in ['degree']:
    network_file_name = "{}_{}".format(data_name, weight_type)

    torch_FINDER_nodefile_degree_scores = None
    FINDER_torch_file_path = os.path.join(".", "code", "results", "FINDER_{}_cost".format(experiment_type), 'real', 'Data_{}'.format(weight_type),\
                                'StepRatio_0.0100', '{}.txt'.format(network_file_name))

    G = nx.read_gml(network_file_path, destringizer=int)

    curr_sccores, curr_costs = cost_calc_score_from_nodes_file(G, FINDER_torch_file_path, None, k=k, write=False, start_idx=0, read_file_inverse=read_file_inverse)
    for i in range(len(curr_costs)): curr_costs[i] =  curr_costs[i]/SUM_COST
    cost_exp_dict["torch_{}_scores_nodefile".format(weight_type)] = [curr_costs, curr_sccores]

    
for weight_type in ['degree']:
    network_file_name = "{}_{}".format(data_name, weight_type)

    torch_FINDER_nodefile_degree_scores = None
    FINDER_torch_file_path = os.path.join(".", "code", "results", "old_FINDER_{}_cost_tf".format(experiment_type), 'real', 'Data_{}'.format(weight_type),\
                                'StepRatio_0.0100', '{}.txt'.format(network_file_name))

    G = nx.read_gml(network_file_path, destringizer=int)

    curr_sccores, curr_costs = cost_calc_score_from_nodes_file(G, FINDER_torch_file_path, None, k=k, write=False, start_idx=0, read_file_inverse=read_file_inverse)
    for i in range(len(curr_costs)): curr_costs[i] =  curr_costs[i]/SUM_COST
    cost_exp_dict["tf_{}_scores_nodefile".format(weight_type)] = [curr_costs, curr_sccores]


from matplotlib import pyplot as plt

plt.figure(figsize=(16,8))

for key in cost_exp_dict.keys():
    plt.plot(cost_exp_dict[key][0], cost_exp_dict[key][1], label=key)

plt.legend()
plt.xlabel("fraction of cost")
plt.ylabel("Graph connectivity {}".format(experiment_type))<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13255451713395638, 'fraction_numerical': 0.00794392523364486, 'mean_word_length': 3.7527757216876387, 'pattern_counts': {'":': 1, '<': 5, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '399199', 'license': 'MIT', 'n_tokens_mistral': 2418, 'n_tokens_neox': 2343, 'n_words': 394, 'path': 'Baselines.ipynb', 'repo_name': 'faraz2023/FINDER-pytorch', 'size': '6420'}