<commit_before># NetworKit native classes and functions
from _NetworKit import GraphProperties, ConnectedComponents

# other submodules
import community
import termgraph
import auxiliary
import nxadapter
import powerlaw

# other modules
import textwrap
import collections
import math
import logging

try:
	import networkx as nx
except ImportError:
	logging.warning("""WARNING: module 'networkx' not installed, which is required by some
						functions.""")
try:
	import tabulate
except ImportError:
	logging.warning("""WARNING: module 'tabulate' not installed, which is required by some
						functions. In order to install 'tabulate', Python 3.3 is required""")

try:
	from scipy import stats
except ImportError:
	logging.warning("""WARNING: module 'scipy' not installed, which is required by some
						functions.""")


########  PROPERTIES ########

def size(G):
	""" Return number of nodes and number of edges"""
	n = G.numberOfNodes()
	m = G.numberOfEdges()
	return (n, m)

def degrees(G):
	""" Return min/max/avg degree"""
	minMaxDeg = GraphProperties.minMaxDegree(G)
	avgDeg = GraphProperties.averageDegree(G)
	return (minMaxDeg[0], minMaxDeg[1], avgDeg)

def degreeDistribution(G):
	""" Return the degree distribution of a graph"""
	return GraphProperties.degreeDistribution(G)


def density(G):
	""" Return the density of the graph"""
	(n, m) = size(G)
	return (2 * m) / (n * (n-1))

def components(G):
	""" Find and analyze detected components """
	logging.info("[...] finding connected components....")
	cc = ConnectedComponents()
	cc.run(G)
	nComponents = cc.numberOfComponents()
	componentSizes = cc.getComponentSizes()
	return (nComponents, componentSizes)

def clustering(G):
	""" Get clustering properties of the graph:
		Returns average local clustering coefficient
	"""
	return GraphProperties.averageLocalClusteringCoefficient(G)


def powerLawExponent(G, dd=None):
	""" Supposing that the degreee distribution fits a power law,
		get the exponent (gamma).
	"""
	if not dd:
		dd = degreeDistribution(G)
	fit = powerlaw.Fit(dd)
	return fit.alpha

def powerLawFit(G, dd=None):
	""" Check if a power law is a good fit for the degree distribution.
	"""
	if not dd:
		dd = degreeDistribution(G)
	fit = powerlaw.Fit(dd)
	R, p = fit.distribution_compare("power_law", "exponential", normalized_ratio=True)
	return ((R > 0), R)


# def powerLawExponent_(G):
#     """ Estimate power law exponent as a linear regression line through a log-log plot
#     of the degree distribution"""
#     def logarithmic(data):
#     	return [(math.log(x) if x != 0 else 0) for x in data]

#     dd = degreeDistribution(G)
#     degrange = range(len(dd))
#     (slope,_,_,_,_) = stats.linregress(logarithmic(degrange), logarithmic(dd))
#     gamma = -1 * slope
#     return gamma
    
	


def properties(nkG, settings):
	nxG = None
	if settings["networkx"]:
		logging.info("[...] converting to NetworX.Graph for some properties....")
		nxG = nxadapter.nk2nx(nkG)

	logging.info("[...] calculating properties")
	
	# size
	n, m = size(nkG)    # n and m

	logging.info("[...] determining degree distribution")
	# degrees 
	degDist = GraphProperties.degreeDistribution(nkG) 
	minDeg, maxDeg, avgDeg = degrees(nkG)

	plfit = powerLawFit(nkG, degDist)
	gamma = powerLawExponent(nkG, degDist)

	# number of isolated nodes
	isolates = degDist[0] if len(degDist) > 0 else None
	satellites = degDist[1] if len(degDist) > 1 else None

	# number of self-loops
	loops = len([(u, v) for (u,v) in nkG.edges() if (u == v)])
	
	# density
	dens = density(nkG)



	# community detection

	ncomPLP, modPLP = None, None
	ncomPLM, modPLM = None, None
	if settings["communities"]:
		logging.info("[...] detecting communities")
		# perform PLP and PLM community detection
		logging.debug("[...] performing community detection: PLP")
		# TODO: avoid printout of status bar
		plp = community.PLP()
		zetaPLP = plp.run(nkG)
		ncomPLP = zetaPLP.numberOfClusters()
		modPLP = community.Modularity().getQuality(zetaPLP, nkG)
		logging.info("[...] performing community detection: PLM")
		plm = community.PLM("balanced")
		zetaPLM = plm.run(nkG)
		ncomPLM = zetaPLM.numberOfClusters()
		modPLM = community.Modularity().getQuality(zetaPLM, nkG)

	# degree histogram
	
	labels, histo = None, None
	if settings["degreeHistogram"]:
		logging.info("[...] preparing degree histogram")    
		histo = degDist
		(labels, histo) = compressHistogram(histo, nbins=25)

	# connected components
	nComponents, componentSizes = None, None
	if settings["components"]:
		nComponents, componentSizes = components(nkG)

	# diameter
	dia = None
	if settings["diameter"] and (n < 1000) and (nComponents is 1):
		logging.info("calculating diameter...")
		dia = nx.diameter(nxG)


	# calculate eccentricity
	ecc = None
	if settings["eccentricity"] and (n < 1000) and (nComponents is 1):
		logging.info("calculating eccentricity...")
		eccentricities = nx.eccentricity(nxG)
		ecc = sum(val for val in eccentricities.values()) / n


	# clustering
	avglcc = None
	if settings["clustering"]:
		avglcc = GraphProperties.averageLocalClusteringCoefficient(nkG)

	# degree assortativity
	assort = None
	if settings["assortativity"]:
		assort = nx.degree_assortativity_coefficient(nxG)




	# betweenness centrality
	# TODO: average betweenness centrality?

	props = {
		 "name": nkG.getName(),
		 "n": n,
		 "m": m,
		 "minDeg": minDeg,
		 "maxDeg": maxDeg,
		 "avgDeg": avgDeg,
		 "plfit": plfit,
		 "gamma": gamma,
		 "avglcc": avglcc,
		 "nComponents": nComponents,
		 "sizeLargestComponent": max(componentSizes.values()),
		 "dia": dia,
		 "ecc": ecc,
		 "isolates": isolates,
		 "loops": loops,
		 "ncomPLP": ncomPLP,
		 "modPLP": modPLP,
		 "ncomPLM": ncomPLM,
		 "modPLM": modPLM,
		 "dens": dens,
		 "assort": assort,
		 "histo": (labels, histo),
		 }

	return props


def overview(nkG, settings=collections.defaultdict(lambda: True)):
	"""
	Print an overview of important network properties to the terminal.
	"""
	props = properties(nkG, settings)
	basicProperties = [
		["nodes (n)", props["n"]],
		["edges (m)", props["m"]],
		["min. degree", props["minDeg"]],
		["max. degree", props["maxDeg"]],
		["avg. degree", "{0:.6f}".format(props["avgDeg"])],
		["degree power law fit?", "{0}, {1}".format(props["plfit"][0], "{0:.6f}".format(props["plfit"][1]))],
		["degree power law exponent", "{0:.4f}".format(props["gamma"])],
		["isolated nodes", props["isolates"]],
		["self-loops", props["loops"]],
		["density", "{0:.6f}".format(props["dens"]) if props["dens"] else None]
	]
	pathStructure = [
		["connected components", props["nComponents"]],
		["size of largest component", props["sizeLargestComponent"]],
		["diameter", props["dia"]],
		["avg. eccentricity", props["ecc"]],
	]
	
	miscProperties = [<commit_msg>power law exponent not shown if power law does not fit<commit_after># NetworKit native classes and functions
from _NetworKit import GraphProperties, ConnectedComponents

# other submodules
import community
import termgraph
import auxiliary
import nxadapter
import powerlaw

# other modules
import textwrap
import collections
import math
import logging

try:
	import networkx as nx
except ImportError:
	logging.warning("""WARNING: module 'networkx' not installed, which is required by some
						functions.""")
try:
	import tabulate
except ImportError:
	logging.warning("""WARNING: module 'tabulate' not installed, which is required by some
						functions. In order to install 'tabulate', Python 3.3 is required""")

try:
	from scipy import stats
except ImportError:
	logging.warning("""WARNING: module 'scipy' not installed, which is required by some
						functions.""")


########  PROPERTIES ########

def size(G):
	""" Return number of nodes and number of edges"""
	n = G.numberOfNodes()
	m = G.numberOfEdges()
	return (n, m)

def degrees(G):
	""" Return min/max/avg degree"""
	minMaxDeg = GraphProperties.minMaxDegree(G)
	avgDeg = GraphProperties.averageDegree(G)
	return (minMaxDeg[0], minMaxDeg[1], avgDeg)

def degreeDistribution(G):
	""" Return the degree distribution of a graph"""
	return GraphProperties.degreeDistribution(G)


def density(G):
	""" Return the density of the graph"""
	(n, m) = size(G)
	return (2 * m) / (n * (n-1))

def components(G):
	""" Find and analyze detected components """
	logging.info("[...] finding connected components....")
	cc = ConnectedComponents()
	cc.run(G)
	nComponents = cc.numberOfComponents()
	componentSizes = cc.getComponentSizes()
	return (nComponents, componentSizes)

def clustering(G):
	""" Get clustering properties of the graph:
		Returns average local clustering coefficient
	"""
	return GraphProperties.averageLocalClusteringCoefficient(G)


def powerLawExponent(G, dd=None):
	""" Supposing that the degreee distribution fits a power law,
		get the exponent (gamma).
	"""
	if not dd:
		dd = degreeDistribution(G)
	fit = powerlaw.Fit(dd)
	return fit.alpha

def powerLawFit(G, dd=None):
	""" Check if a power law is a good fit for the degree distribution.
	"""
	if not dd:
		dd = degreeDistribution(G)
	fit = powerlaw.Fit(dd)
	R, p = fit.distribution_compare("power_law", "exponential", normalized_ratio=True)
	return ((R > 0), R)


# def powerLawExponent_(G):
#     """ Estimate power law exponent as a linear regression line through a log-log plot
#     of the degree distribution"""
#     def logarithmic(data):
#     	return [(math.log(x) if x != 0 else 0) for x in data]

#     dd = degreeDistribution(G)
#     degrange = range(len(dd))
#     (slope,_,_,_,_) = stats.linregress(logarithmic(degrange), logarithmic(dd))
#     gamma = -1 * slope
#     return gamma
    
	


def properties(nkG, settings):
	nxG = None
	if settings["networkx"]:
		logging.info("[...] converting to NetworX.Graph for some properties....")
		nxG = nxadapter.nk2nx(nkG)

	logging.info("[...] calculating properties")
	
	# size
	n, m = size(nkG)    # n and m

	logging.info("[...] determining degree distribution")
	# degrees 
	degDist = GraphProperties.degreeDistribution(nkG) 
	minDeg, maxDeg, avgDeg = degrees(nkG)

	plfit = powerLawFit(nkG, degDist)
	gamma = powerLawExponent(nkG, degDist)

	# number of isolated nodes
	isolates = degDist[0] if len(degDist) > 0 else None
	satellites = degDist[1] if len(degDist) > 1 else None

	# number of self-loops
	loops = len([(u, v) for (u,v) in nkG.edges() if (u == v)])
	
	# density
	dens = density(nkG)



	# community detection

	ncomPLP, modPLP = None, None
	ncomPLM, modPLM = None, None
	if settings["communities"]:
		logging.info("[...] detecting communities")
		# perform PLP and PLM community detection
		logging.debug("[...] performing community detection: PLP")
		# TODO: avoid printout of status bar
		plp = community.PLP()
		zetaPLP = plp.run(nkG)
		ncomPLP = zetaPLP.numberOfClusters()
		modPLP = community.Modularity().getQuality(zetaPLP, nkG)
		logging.info("[...] performing community detection: PLM")
		plm = community.PLM("balanced")
		zetaPLM = plm.run(nkG)
		ncomPLM = zetaPLM.numberOfClusters()
		modPLM = community.Modularity().getQuality(zetaPLM, nkG)

	# degree histogram
	
	labels, histo = None, None
	if settings["degreeHistogram"]:
		logging.info("[...] preparing degree histogram")    
		histo = degDist
		(labels, histo) = compressHistogram(histo, nbins=25)

	# connected components
	nComponents, componentSizes = None, None
	if settings["components"]:
		nComponents, componentSizes = components(nkG)

	# diameter
	dia = None
	if settings["diameter"] and (n < 1000) and (nComponents is 1):
		logging.info("calculating diameter...")
		dia = nx.diameter(nxG)


	# calculate eccentricity
	ecc = None
	if settings["eccentricity"] and (n < 1000) and (nComponents is 1):
		logging.info("calculating eccentricity...")
		eccentricities = nx.eccentricity(nxG)
		ecc = sum(val for val in eccentricities.values()) / n


	# clustering
	avglcc = None
	if settings["clustering"]:
		avglcc = GraphProperties.averageLocalClusteringCoefficient(nkG)

	# degree assortativity
	assort = None
	if settings["assortativity"]:
		assort = nx.degree_assortativity_coefficient(nxG)




	# betweenness centrality
	# TODO: average betweenness centrality?

	props = {
		 "name": nkG.getName(),
		 "n": n,
		 "m": m,
		 "minDeg": minDeg,
		 "maxDeg": maxDeg,
		 "avgDeg": avgDeg,
		 "plfit": plfit,
		 "gamma": gamma,
		 "avglcc": avglcc,
		 "nComponents": nComponents,
		 "sizeLargestComponent": max(componentSizes.values()),
		 "dia": dia,
		 "ecc": ecc,
		 "isolates": isolates,
		 "loops": loops,
		 "ncomPLP": ncomPLP,
		 "modPLP": modPLP,
		 "ncomPLM": ncomPLM,
		 "modPLM": modPLM,
		 "dens": dens,
		 "assort": assort,
		 "histo": (labels, histo),
		 }

	return props


def overview(nkG, settings=collections.defaultdict(lambda: True)):
	"""
	Print an overview of important network properties to the terminal.
	"""
	props = properties(nkG, settings)
	basicProperties = [
		["nodes (n)", props["n"]],
		["edges (m)", props["m"]],
		["min. degree", props["minDeg"]],
		["max. degree", props["maxDeg"]],
		["avg. degree", "{0:.6f}".format(props["avgDeg"])],
		["degree power law fit?", "{0}, {1}".format(props["plfit"][0], "{0:.6f}".format(props["plfit"][1]))],
		["degree power law exponent", "{0:.4f}".format(props["gamma"]) if props["plfit"][0] else None],
		["isolated nodes", props["isolates"]],
		["self-loops", props["loops"]],
		["density", "{0:.6f}".format(props["dens"]) if props["dens"] else None]
	]
	pathStructure = [
		["connected components", props["nComponents"]],
		["size of largest component", props["sizeLargestComponent"]],
		["diameter", props["dia"]],
		["avg. eccentricity", props["ecc"]],
	]
	
	miscProperties = [
{'fraction_non_alphanumeric': 0.14641331565768154, 'fraction_numerical': 0.005818235380762999, 'mean_word_length': 4.425089892129445, 'pattern_counts': {'":': 44, '<': 7, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4584640', 'n_tokens_mistral': 5016, 'n_tokens_neox': 4456, 'n_words': 1274}