<commit_before>import random
import linecache
from unidecode import unidecode

# Process links into list
finallist = [None] * 5716809
with open('links-simple-sorted.txt', 'r') as src:
	for line in src:
		[oNode, dNode] = line.split(':')
		finallist[int(oNode)] = dNode.rstrip('\n')[1:]

##########################################################################################
# Analyze basic properties of a particular node (randomly selected) 

# Get a random node, and pull that line from the links doc; want this to be an option
# oNode = random.randint(1,5706070)
# dNode = finallist[oNode]
# dNode = dNode.split(' ')
# Translate these into titles and print the result
# oname = linecache.getline('titles-sorted.txt',int(oNode))
# oname = oname[:-1] # Gets rid of the trailing newline
# print '\nORIGIN NODE: ' + oname + '\n'
# print 'DESTINATION NODES:'
# for thisnum in dNode:
# 	dname = linecache.getline('titles-sorted.txt',int(thisnum))[:-1]
# 	print '     ' + dname
# print '\n'

##########################################################################################
# Connectivity across the whole network

# Calculate average number of 1st-order connections
totalConn = 0
linksout = []
for node in finallist:
	if node != None: # If the oNode has outlinks
		totalConn += len(node.split(' '))
		linksout += [len(node.split(' '))]
	else:
		linksout += [0]
avgConn = float(totalConn/5716809)
print avgConn
print max(linksout)

##########################################################################################
# Average clustering coefficient

for oNode in range(1,5716809+1):
	oname = linecache.getline('titles-sorted.txt',int(oNode))
	dNode = finallist[oNode].split(' ')

	# Gather the same info for each of the neighbors
	# Make this a function, can do recursive calls to check multiple depths?
	twoSteps = ''
	for neighbor in dNode:
		if dNode != ['']:
			twoSteps += finallist[int(neighbor)]
	twoSteps = twoSteps.split(' ')

	# Have to find the number of overlaps
	oneANDtwo = 0
	for firstStep in dNode:
		oneANDtwo += twoSteps.count(firstStep)


	if len(dNode) > 1:
		# CLUSTERING COEFFICIENT
		ai = float(len(dNode))
		Ei = float(oneANDtwo)
		# Calculate the coefficient
		clCoeff = (2*Ei)/(ai*(ai-1))
		print 'Clustering Coefficient: ' + str(clCoeff)<commit_msg>Fixed type comparison in analysis.py<commit_after>import random
import linecache
from unidecode import unidecode

# Process links into list
finallist = [None] * 5716809
with open('links-simple-sorted.txt', 'r') as src:
	for line in src:
		[oNode, dNode] = line.split(':')
		finallist[int(oNode)] = dNode.rstrip('\n')[1:]

##########################################################################################
# Analyze basic properties of a particular node (randomly selected) 

# Get a random node, and pull that line from the links doc; want this to be an option
# oNode = random.randint(1,5706070)
# dNode = finallist[oNode]
# dNode = dNode.split(' ')
# Translate these into titles and print the result
# oname = linecache.getline('titles-sorted.txt',int(oNode))
# oname = oname[:-1] # Gets rid of the trailing newline
# print '\nORIGIN NODE: ' + oname + '\n'
# print 'DESTINATION NODES:'
# for thisnum in dNode:
# 	dname = linecache.getline('titles-sorted.txt',int(thisnum))[:-1]
# 	print '     ' + dname
# print '\n'

##########################################################################################
# Connectivity across the whole network

# Calculate average number of 1st-order connections
totalConn = 0
linksout = []
for node in finallist:
	if type(node) != type(None): # If the oNode has outlinks
		totalConn += len(node.split(' '))
		linksout += [len(node.split(' '))]
	else:
		linksout += [0]
avgConn = float(totalConn/5716809)
print avgConn # Average outgoing connectivity
print max(linksout) # Maximum number of links
print totalConn # Total number of links in dataset
{'fraction_non_alphanumeric': 0.22871108824286082, 'fraction_numerical': 0.01775147928994083, 'mean_word_length': 5.14218009478673, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3540940', 'n_tokens_mistral': 1312, 'n_tokens_neox': 1222, 'n_words': 368}