<commit_before>import sys, os
import networkx as nx
import pdb

NULL_CONCEPT = 1

concepts_dict = {}
def read_concepts(concepts_dict_file):
	for line in concepts_dict_file.readlines():
		concepts_dict[int(line.strip().split()[1])] = line.strip().split()[0].lower()

relations_dict = {}
def read_relations(relations_dict_file):
	for line in relations_dict_file.readlines():
		relations_dict[int(line.strip().split()[1])] = line.strip().split()[0].lower()

def to_nx_graph(all_heads, all_tags, all_concepts):
	#print all_heads
	#print all_tags
	#print all_concepts
	amr_roots = []
	amr_nx_graph = nx.MultiDiGraph()
	for idx in range(1, len(all_concepts)):
		concept = all_concepts[idx]
		if concept == NULL_CONCEPT:
			continue
		amr_nx_graph.add_node(idx, instance=concepts_dict[concept], parents=all_heads[idx])
		if all_heads[idx][0] == 0: #this is the root
			amr_roots.append(idx)
			continue #so don't add any edge
		for i, parent in enumerate(all_heads[idx]):
			amr_nx_graph.add_edge(parent, idx, relation=relations_dict[all_tags[idx][i]])
	return amr_nx_graph, amr_roots

shortname_dict = {}
def get_amr_string(root, amr_nx_graph, tab_levels=1):
	amr_string = ""
	#print amr_nx_graph.successors(root)
	for child in amr_nx_graph.successors(root):
		if not child in shortname_dict.keys():
			size = len(shortname_dict.keys())
			child_amr_string = get_amr_string(child, amr_nx_graph, tab_levels+1)
			shortname_dict[child] = "c"+str(size)
			amr_string += "\t"*tab_levels + "\t:{0} ".format(amr_nx_graph[root][child][0]['relation']) + child_amr_string
		else:
			amr_string += "\t:{0} {1}\n".format(shortname_dict[child], amr_nx_graph[root][child][0]['relation'])

	if not root in shortname_dict.keys():
		size = len(shortname_dict.keys())
		shortname_dict[root] = "c"+str(size)
		amr_string = "({0} / {1}\n".format(shortname_dict[root], amr_nx_graph.node[root]['instance']) + amr_string + ")"
	else:
		amr_string = "{0}".format(amr_nx_graph.node[root]['instance'])
	return amr_string	

def print_nx_graph(nx_graph, amr_roots):
	#print amr_nx_graph.nodes()
	#print amr_nx_graph.edges()	
	#print amr_root
	#pdb.set_trace()
	if len(amr_roots) > 1:
		amr_nx_graph.add_node(0, instance='multi-sentence', parents=None)
		for amr_root in amr_roots:
			amr_nx_graph.add_edge(0, amr_root, relation='op')
		amr_roots = [0]	
	print get_amr_string(amr_roots[0], amr_nx_graph)
	print

if __name__ == "__main__":
	if len(sys.argv) < 2:
		print "usage: vw_pred_to_amr.py <data.pred> <concepts_dict> <relations_dict>"
		sys.exit(0)
	vw_pred_file = open(sys.argv[1], 'r')
	concepts_dict_file = open(sys.argv[2], 'r')
	relations_dict_file = open(sys.argv[3], 'r')
	read_concepts(concepts_dict_file)
	read_relations(relations_dict_file)
	all_heads = [[0]]
	all_tags = [[0]]
	all_concepts = [0]
	global shortname_dict
	for line in vw_pred_file.readlines():
		line = line.strip("\n")<commit_msg>If no concept predicted then print a dummy multi-sentence node - otherwise smatch cries<commit_after>import sys, os
import networkx as nx
import pdb

NULL_CONCEPT = 1

concepts_dict = {}
def read_concepts(concepts_dict_file):
	for line in concepts_dict_file.readlines():
		concepts_dict[int(line.strip().split()[1])] = line.strip().split()[0].lower()

relations_dict = {}
def read_relations(relations_dict_file):
	for line in relations_dict_file.readlines():
		relations_dict[int(line.strip().split()[1])] = line.strip().split()[0].lower()

def to_nx_graph(all_heads, all_tags, all_concepts):
	#print all_heads
	#print all_tags
	#print all_concepts
	amr_roots = []
	amr_nx_graph = nx.MultiDiGraph()
	for idx in range(1, len(all_concepts)):
		concept = all_concepts[idx]
		if concept == NULL_CONCEPT:
			continue
		amr_nx_graph.add_node(idx, instance=concepts_dict[concept], parents=all_heads[idx])
		if all_heads[idx][0] == 0: #this is the root
			amr_roots.append(idx)
			continue #so don't add any edge
		for i, parent in enumerate(all_heads[idx]):
			amr_nx_graph.add_edge(parent, idx, relation=relations_dict[all_tags[idx][i]])
	return amr_nx_graph, amr_roots

shortname_dict = {}
def get_amr_string(root, amr_nx_graph, tab_levels=1):
	amr_string = ""
	#print amr_nx_graph.successors(root)
	for child in amr_nx_graph.successors(root):
		if not child in shortname_dict.keys():
			size = len(shortname_dict.keys())
			child_amr_string = get_amr_string(child, amr_nx_graph, tab_levels+1)
			shortname_dict[child] = "c"+str(size)
			amr_string += "\t"*tab_levels + "\t:{0} ".format(amr_nx_graph[root][child][0]['relation']) + child_amr_string
		else:
			amr_string += "\t:{0} {1}\n".format(shortname_dict[child], amr_nx_graph[root][child][0]['relation'])

	if not root in shortname_dict.keys():
		size = len(shortname_dict.keys())
		shortname_dict[root] = "c"+str(size)
		amr_string = "({0} / {1}\n".format(shortname_dict[root], amr_nx_graph.node[root]['instance']) + amr_string + ")"
	else:
		amr_string = "{0}".format(amr_nx_graph.node[root]['instance'])
	return amr_string	

def print_nx_graph(nx_graph, amr_roots):
	#print amr_nx_graph.nodes()
	#print amr_nx_graph.edges()	
	#print amr_root
	#pdb.set_trace()
	if not amr_roots: #Only null concepts predicted
		amr_nx_graph.add_node(0, instance='multi-sentence', parents=None)
		print get_amr_string(0, amr_nx_graph)
	elif len(amr_roots) > 1:
		amr_nx_graph.add_node(0, instance='multi-sentence', parents=None)
		for amr_root in amr_roots:
			amr_nx_graph.add_edge(0, amr_root, relation='op')
		print get_amr_string(0, amr_nx_graph)
	else:	
		print get_amr_string(amr_roots[0], amr_nx_graph)
	print

if __name__ == "__main__":
	if len(sys.argv) < 2:
		print "usage: vw_pred_to_amr.py <data.pred> <concepts_dict> <relations_dict>"
		sys.exit(0)
	vw_pred_file = open(sys.argv[1], 'r')
	concepts_dict_file = open(sys.argv[2], 'r')
	relations_dict_file = open(sys.argv[3], 'r')
	read_concepts(concepts_dict_file)
	read_relations(relations_dict_file)
	all_heads = [[0]]
	all_tags = [[0]]
	all_concepts = [0]
	global shortname_dict
	for line in vw_pred_file.readlines():
		line = line.strip("\n")
{'fraction_non_alphanumeric': 0.14610659139963472, 'fraction_numerical': 0.010625933919973435, 'mean_word_length': 6.078730904817862, 'pattern_counts': {'":': 2, '<': 11, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2210522', 'n_tokens_mistral': 2556, 'n_tokens_neox': 2385, 'n_words': 430}