<commit_before>#!/usr/bin/env python
import json
import os.path
import argparse
from functools import partial

def read_results(file_name):
	if not os.path.isfile(file_name):
		print 'No crawling file found! Make sure you run crawler.py first!'
		return {}
	try:
		with open(file_name, 'r') as filestream:
			return json.load(filestream)
	except ValueError:
		print 'Couldn\'t parse crawler file'
		return {}

def write_graph(graph):
	with open('graph.json', 'w') as filestream:
		filestream.write(json.dumps(graph, indent = 4, separators = (',', ': ')))

def add_or_increment_edge(edge_dict, node1, node2):
	edge = None
	if (node1, node2) in edge_dict:
		edge = edge_dict[(node1, node2)]
	elif (node2, node1) in edge_dict:
		edge = edge_dict[(node2, node1)]
	
	if not edge:
		edge = { 'from' : node1, 'to' : node2, 'value' : 1 }
		edge_dict[(node1, node2)] = edge
	else:
		edge['value'] += 1

	return edge

def build_nodes(edge_list):
	node_dict = {}
	for edge in edge_list:
		sub1 = edge['from']
		sub2 = edge['to']

		if sub1 not in node_dict:
			node_dict[sub1] = {'index' : len(node_dict), 'label' : sub1}
		if sub2 not in node_dict:
			node_dict[sub2] = {'index' : len(node_dict), 'label' : sub2}

		edge['from'] = node_dict[sub1]['index']
		edge['to'] = node_dict[sub2]['index']
		
	return node_dict.values()

# selects edges from edge_list such that the maximum number of subreddits doesn't exceed max_subs
def trim_to_size(edge_list, max_subs):
	subs = set()
	trimmed_edge_list = []
	
	for edge in edge_list:
		if len(subs) >= max_subs and (edge['from'] not in subs or edge['to'] not in subs):
			continue

		subs.add(edge['from'])
		subs.add(edge['to'])
		trimmed_edge_list.append(edge)
	
	return trimmed_edge_list


def build_matrix(node_list, edge_list):
	size = len(node_list)
	#create size x size empty matrix
	edge_matrix = [[0] * size for i in range(size)]

	for edge in edge_list:
		edge_matrix[edge['from']][edge['to']] = edge['value']
		edge_matrix[edge['to']][edge['from']] = edge['value']

	return edge_matrix

def construct_graph(file_name, filter_min, max_subs):
	crawl_results = read_results(file_name)
	
	if not crawl_results:
		return
	
	edge_dict = {}

	for user_key in crawl_results:
		user_subs = crawl_results[user_key]
		if '___last_seen' in user_subs:
			del user_subs['___last_seen']

		for subreddit1 in user_subs:
			for subreddit2 in user_subs:
				if subreddit1 is subreddit2:
					continue

				add_or_increment_edge(edge_dict, subreddit1, subreddit2)

	edge_list = edge_dict.values()
	edge_list.sort(key = lambda l: l['value'])

	if filter_min > 1:
		print 'filtering results below ' + str(filter_min)
		edge_list = [edge for edge in edge_list if edge['value'] >= filter_min]

	if max_subs > 0:
		print 'limiting amount of subs to ' + str(max_subs)
		edge_list = trim_to_size(edge_list, max_subs)

	node_list = build_nodes(edge_list)

	edge_matrix = build_matrix(node_list, edge_list)

	# construct empty graph object with nodes and edges
	graph_result = { 'groups' : node_list, 'matrix' : edge_matrix }

	print 'created graph file for ' + str(len(graph_result['groups'])) + ' subreddits'
	print 'no. of edges is ' + str(len(edge_list))

	print 'writing'
	write_graph(graph_result)

if __name__ == '__main__':
	parser = argparse.ArgumentParser(description='Transform reddit crawling results into a graphing result for vis.js')
	parser.add_argument('--file', action='store',  default='results.json', help='file to read crawling results from')
	parser.add_argument('--min', action='store', default=1, help='Minumum number of users each relation needs for it to be added to the graph')
	parser.add_argument('--subs', action='store', default=0, help='Maximum number of subs to include')
	args = parser.parse_args()

	construct_graph(args.file, int(args.min), int(args.subs))
<commit_msg>Added normalization<commit_after>#!/usr/bin/env python
import json
import os.path
import argparse
from functools import partial

def read_results(file_name):
	if not os.path.isfile(file_name):
		print 'No crawling file found! Make sure you run crawler.py first!'
		return {}
	try:
		with open(file_name, 'r') as filestream:
			return json.load(filestream)
	except ValueError:
		print 'Couldn\'t parse crawler file'
		return {}

def write_graph(graph):
	with open('graph.json', 'w') as filestream:
		filestream.write(json.dumps(graph, indent = 4, separators = (',', ': ')))

def add_or_increment_edge(edge_dict, node1, node2):
	edge = None
	if (node1, node2) in edge_dict:
		edge = edge_dict[(node1, node2)]
	elif (node2, node1) in edge_dict:
		edge = edge_dict[(node2, node1)]
	
	if not edge:
		edge = { 'from' : node1, 'to' : node2, 'fromCount' : 1, 'toCount' : 1}
		edge_dict[(node1, node2)] = edge
	else:
		edge['fromCount'] += 1
		edge['toCount'] += 1

def build_nodes(edge_list):
	node_dict = {}
	
	for edge in edge_list:
	
		if edge['from'] not in node_dict:
			node_dict[edge['from']] = {'index' : len(node_dict), 'label' : edge['from']}
		if edge['to'] not in node_dict:
			node_dict[edge['to']] = {'index' : len(node_dict), 'label' : edge['to']}

		edge['from'] = node_dict[edge['from']]['index']
		edge['to'] = node_dict[edge['to']]['index']

	return node_dict.values()

# selects edges from edge_list such that the maximum number of subreddits doesn't exceed max_subs
def trim_to_size(edge_list, max_subs):
	subs = set()
	trimmed_edge_list = []
	
	for edge in edge_list:
		if len(subs) >= max_subs and (edge['from'] not in subs or edge['to'] not in subs):
			continue

		subs.add(edge['from'])
		subs.add(edge['to'])
		trimmed_edge_list.append(edge)
	
	return trimmed_edge_list

def normalize_edges(edge_list):
	node_dict = {}
	for edge in edge_list:
		node_from = node_dict.get(edge['from'], 0)
		node_to = node_dict.get(edge['to'], 0)
		
		node_from += edge['fromCount']
		node_to += edge['toCount']

		node_dict[edge['from']] = node_from
		node_dict[edge['to']] = node_to

	for edge in edge_list:
		edge['fromCount'] = edge['fromCount'] / float(node_dict[edge['from']])
		edge['toCount'] = edge['toCount'] / float(node_dict[edge['to']])

def build_matrix(node_list, edge_list):
	size = len(node_list)
	#create size x size empty matrix
	edge_matrix = [[0] * size for i in range(size)]

	for edge in edge_list:
		edge_matrix[edge['from']][edge['to']] = edge['fromCount']
		edge_matrix[edge['to']][edge['from']] = edge['toCount']

	return edge_matrix

def construct_graph(file_name, filter_min, max_subs, normalize):
	print 'reading results'
	crawl_results = read_results(file_name)
	
	if not crawl_results:
		return
	
	edge_dict = {}

	for user_key in crawl_results:
		user_subs = crawl_results[user_key]
		if '___last_seen' in user_subs:
			del user_subs['___last_seen']

		for subreddit1 in user_subs:
			for subreddit2 in user_subs:
				if subreddit1 is subreddit2:
					continue

				add_or_increment_edge(edge_dict, subreddit1, subreddit2)

	edge_list = edge_dict.values()
	edge_list.sort(key = lambda l: l['fromCount'])

	if filter_min > 1:
		print 'filtering results below ' + str(filter_min)
		edge_list = [edge for edge in edge_list if edge['fromCount'] >= filter_min]

	if max_subs > 0:
		print 'limiting amount of subs to ' + str(max_subs)
		edge_list = trim_to_size(edge_list, max_subs)

	if normalize:
		print 'normalizing results'
		normalize_edges(edge_list)

	node_list = build_nodes(edge_list)

	edge_matrix = build_matrix(node_list, edge_list)

	# construct empty graph object with nodes and edges
	graph_result = { 'groups' : node_list, 'matrix' : edge_matrix }

	print 'created graph file for ' + str(len(graph_result['groups'])) + ' subreddits'
	print 'no. of edges is ' + str(len(edge_list))

	print 'writing'
	write_graph(graph_result)

if __name__ == '__main__':
	parser = argparse.ArgumentParser(description='Transform reddit crawling results into a graphing result for vis.js')
	parser.add_argument('--file', action='store',  default='results.json', help='file to read crawling results from')
	parser.add_argument('--min', action='store', default=1, help='Minumum number of users each relation needs for it to be added to the graph')
	parser.add_argument('--subs', action='store', default=0, help='Maximum number of subs to include')
	parser.add_argument('--normalize', action='store_true', help='Normalizes the results, making the input of each node the same size')
	args = parser.parse_args()

	construct_graph(args.file, int(args.min), int(args.subs), args.normalize)

{'fraction_non_alphanumeric': 0.133554738465188, 'fraction_numerical': 0.008302692444549875, 'mean_word_length': 4.980141843971631, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1015151', 'n_tokens_mistral': 3194, 'n_tokens_neox': 2980, 'n_words': 791}