<commit_before>#@title Imports
import os
import numpy as np
import time

import graph_tool


def gini(array):
    array = array.astype(np.float32)
    array += np.finfo(np.float32).eps
    array = np.sort(array)
    n = array.shape[0]
    index = np.arange(1, n + 1)
    return np.sum((2 * index - n  - 1) * array) / (n * np.sum(array))


def diameter_lb(graph, n_tries=100):
  max_diameter = -1
  nodes = np.random.choice(list(graph.vertices()), n_tries)
  for node in nodes:
    diameter = graph_tool.topology.pseudo_diameter(graph, node)[0]
    if diameter > max_diameter:
      max_diameter = diameter
  return max_diameter


def bfs_sp(graph, source, others):
  others = set(others)
  for edge in graph_tool.search.bfs_iterator(graph, source):
    if edge.target() in others:
      return graph_tool.topology.shortest_distance(graph, source, edge.target())
  return -1


def average_cc_sp_length(G_comp):
  sp_lengths = 0
  sp_distances = graph_tool.topology.shortest_distance(G_comp, directed=False)
  for vtx in G_comp.vertices():
    sp_lengths += np.sum(sp_distances[vtx].a)
  return sp_lengths / G_comp.num_vertices() / (G_comp.num_vertices() - 1)


def matrix_row_norm(X):
  return (X.T / np.linalg.norm(X, axis=1)).T


def GraphMetrics(G):
  result = {}
  result['n_nodes'] = G.num_vertices()
  result['n_edges'] = G.num_edges()
  result['edge_density'] = G.num_edges() / G.num_vertices() / (G.num_vertices() - 1)
  result['avg_in_degree'] = np.mean(G.get_in_degrees(G.get_vertices()))
  result['edge_reciprocity'] = graph_tool.topology.edge_reciprocity(G)
  result['avg_undirected_degree'] = np.mean(G.get_out_degrees(G.get_vertices())) / 2
  result['degree_gini'] = gini(G.get_out_degrees(G.get_vertices()))
  result['pseudo_diameter'] = diameter_lb(G)
  coreness = np.array(graph_tool.topology.kcore_decomposition(G).a)
  result['coreness_eq_1'] = np.sum(coreness == 1) / G.num_vertices()
  result['coreness_geq_2'] = np.sum(coreness >= 2) / G.num_vertices()
  result['coreness_geq_5'] = np.sum(coreness >= 5) / G.num_vertices()
  result['coreness_geq_10'] = np.sum(coreness >= 10) / G.num_vertices()
  result['coreness_gini'] = gini(coreness)
  result['avg_local_cc'] = np.mean(np.array(graph_tool.clustering.local_clustering(G).a))
  result['global_cc'] = graph_tool.clustering.global_clustering(G)[0]<commit_msg>Avoid double transpose for normalization<commit_after>#@title Imports
import os
import numpy as np
import time

import graph_tool


def gini(array):
    array = array.astype(np.float32)
    array += np.finfo(np.float32).eps
    array = np.sort(array)
    n = array.shape[0]
    index = np.arange(1, n + 1)
    return np.sum((2 * index - n  - 1) * array) / (n * np.sum(array))


def diameter_lb(graph, n_tries=100):
  max_diameter = -1
  nodes = np.random.choice(list(graph.vertices()), n_tries)
  for node in nodes:
    diameter = graph_tool.topology.pseudo_diameter(graph, node)[0]
    if diameter > max_diameter:
      max_diameter = diameter
  return max_diameter


def bfs_sp(graph, source, others):
  others = set(others)
  for edge in graph_tool.search.bfs_iterator(graph, source):
    if edge.target() in others:
      return graph_tool.topology.shortest_distance(graph, source, edge.target())
  return -1


def average_cc_sp_length(G_comp):
  sp_lengths = 0
  sp_distances = graph_tool.topology.shortest_distance(G_comp, directed=False)
  for vtx in G_comp.vertices():
    sp_lengths += np.sum(sp_distances[vtx].a)
  return sp_lengths / G_comp.num_vertices() / (G_comp.num_vertices() - 1)


def matrix_row_norm(X):
  return X / np.linalg.norm(X, axis=1)[:, None]


def GraphMetrics(G):
  result = {}
  result['n_nodes'] = G.num_vertices()
  result['n_edges'] = G.num_edges()
  result['edge_density'] = G.num_edges() / G.num_vertices() / (G.num_vertices() - 1)
  result['avg_in_degree'] = np.mean(G.get_in_degrees(G.get_vertices()))
  result['edge_reciprocity'] = graph_tool.topology.edge_reciprocity(G)
  result['avg_undirected_degree'] = np.mean(G.get_out_degrees(G.get_vertices())) / 2
  result['degree_gini'] = gini(G.get_out_degrees(G.get_vertices()))
  result['pseudo_diameter'] = diameter_lb(G)
  coreness = np.array(graph_tool.topology.kcore_decomposition(G).a)
  result['coreness_eq_1'] = np.sum(coreness == 1) / G.num_vertices()
  result['coreness_geq_2'] = np.sum(coreness >= 2) / G.num_vertices()
  result['coreness_geq_5'] = np.sum(coreness >= 5) / G.num_vertices()
  result['coreness_geq_10'] = np.sum(coreness >= 10) / G.num_vertices()
  result['coreness_gini'] = gini(coreness)
  result['avg_local_cc'] = np.mean(np.array(graph_tool.clustering.local_clustering(G).a))
  result['global_cc'] = graph_tool.clustering.global_clustering(G)[0]
{'fraction_non_alphanumeric': 0.1509594882729211, 'fraction_numerical': 0.013219616204690832, 'mean_word_length': 4.706812652068127, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1873449', 'n_tokens_mistral': 1872, 'n_tokens_neox': 1801, 'n_words': 300}