<commit_before>import os, sys
import common
import dot
import argparse
import json
from simprog import Similarity

def generate_graphs(project):
  """Run dljc
  Compile test sources
  Generate program graphs using prog2dfg
  Precompute graph kernels that are independent of ontology stuff
  """
  print("Generating graphs for {0}".format(project))
  common.run_dljc(project,
                  ['graphtool'],
                  ['--graph-jar', common.get_jar('prog2dfg.jar'),
                   '--cache'])

def generate_dtrace(project):
  #TODO: set the out file to common.get_dtrace_file_for_project(project)
  common.run_dljc(project,
                  ['dyntrace'], ['--cache'])  

def gather_kernels(projects, corpus_kernel_file):
  print("Gathering kernels from projects {0}".format(" and ".join(projects)))
  with open(corpus_kernel_file, "w") as corpus_kernel_file_handle:
    for project in projects:
      project_dir = common.get_project_dir(project)
      #print dot.dot_dirs(project)
      out_dir = dot.dot_dirs(project)[0] # only consider the first one
      project_kernel_file_path = dot.get_kernel_path(project, out_dir)
      
      if os.path.isfile(project_kernel_file_path):
        with open(project_kernel_file_path, "r") as fi: 
            corpus_kernel_file_handle.write(fi.read())
      else:
        print ("No kernel file find for project {0}.\n   {1} is not a file.".format(
          project,
          project_kernel_file_path
          ))


def generate_project_kernel(project, cluster_json=None):
  """ run graph kernel computation """
  
  project_dir = common.get_project_dir(project)

  out_dir = dot.dot_dirs(project)[0]
  
  kernel_file_path = dot.get_kernel_path(project, out_dir)
  
  if cluster_json:
    print("Using clustering output for node relabeling:")
    graph_kernel_cmd = ['python',
                        common.get_simprog('precompute_kernel.py'),
                        project_dir,
                        kernel_file_path,
                        cluster_json
                        ]
    common.run_cmd(graph_kernel_cmd, True)
  else:
    graph_kernel_cmd = ['python',
                        common.get_simprog('precompute_kernel.py'),
                        project_dir,
                        kernel_file_path
                        ]
    common.run_cmd(graph_kernel_cmd, True)
    
  print("Generated kernel file for {0} in {1}.".format(project, kernel_file_path))

def get_method_map(project_list):
  dot_to_method_map = {}
  for project in project_list:
    for output_dir in dot.dot_dirs(project):
      #output_dir = dot.dot_dirs(project)[0] # first folder only for now
      method_file = dot.get_method_path(project, output_dir)
      if not os.path.isfile(method_file):
        print ("Cannot find method file for project {0} at {1}".format(project, method_file))
        sys.exit(0)

      with open(method_file, "r") as mf:
        content = mf.readlines()
        for line in content:
          line = line.rstrip()
          items = line.split('\t')
          method_name = items[0]
          method_dot = items[1]
          method_dot_path = dot.get_dot_path(project, output_dir, method_dot)          
          dot_to_method_map[method_dot_path] = method_name
  return dot_to_method_map

def check_similarity(project, result_file, kernel_file, corpus_dot_to_method_map, output_json_file, cluster_json=None, top_k=5):
  """ SUMMARY: use case of the user-driven functionality of PASCALI.
  """

  # fetch various method information from each project in the list
  # output_dir = dot.dot_dirs(project)[0]
  # method_file = dot.get_method_path(project, output_dir)

  # check similarity
  json_result = {}
  sim = Similarity()
  sim.read_graph_kernels(kernel_file)
  iter_num = 3 # number of iteration of the WL-Kernel method
  with open(result_file, "w") as fo:
    for dot_file in corpus_dot_to_method_map.keys():
      dot_method = corpus_dot_to_method_map[dot_file]
      json_result[dot_method] = []
      result_program_list_with_score = sim.find_top_k_similar_graphs(dot_file, dot_file, top_k, iter_num, cluster_json)
      line = dot_file+":\n"
      for (dt, score) in result_program_list_with_score:
        line += "{} , {}\n".format(dt, score)
        if dt not in corpus_dot_to_method_map:
          print("{0} does not exist.".format(dt))
          sys.exit(0)
        json_result[dot_method].append((corpus_dot_to_method_map[dt], score))
      line += "\n"
      fo.write(line)
  with open(output_json_file, "w") as jo:
    json.dump(json_result, jo)

def main():
	
	parser = argparse.ArgumentParser()

	parser.add_argument("-c", "--cluster", type=str, help="Path to the input json file that contains the clustering information.")
  	parser.add_argument("-d", "--dir", type=str, required=True, help="Output folder.")
  	parser.add_argument("-p", "--plist", type=str, help="A comma separated list of projects to work with.")
  	parser.add_argument("-k", "--kernel", action="store_true", help="Recompute kernel vectors.")
  	parser.add_argument("-g", "--graph", action="store_true", help="Regenerate graphs.")

  	args = parser.parse_args()

  	common.mkdir(args.dir)

  	# determine which projects to consider
	project_list = common.get_project_list()
	if args.plist:
		arg_projects = args.plist.split(',')
		project_list = [project for project in project_list if project in arg_projects]

	# determine if need to regerenate graphs
	if args.graph:
		for project in project_list:
			generate_graphs(project)

    # determine if need to recompute the kernel vectors
	if args.kernel:
		for project in project_list:<commit_msg>minor improvement and prettier json output<commit_after>import os, sys
import common
import dot
import argparse
import json
from simprog import Similarity

def generate_graphs(project):
  """Run dljc
  Compile test sources
  Generate program graphs using prog2dfg
  Precompute graph kernels that are independent of ontology stuff
  """
  print("Generating graphs for {0}".format(project))
  common.run_dljc(project,
                  ['graphtool'],
                  ['--graph-jar', common.get_jar('prog2dfg.jar'),
                   '--cache'])

def generate_dtrace(project):
  #TODO: set the out file to common.get_dtrace_file_for_project(project)
  common.run_dljc(project,
                  ['dyntrace'], ['--cache'])  

def gather_kernels(projects, corpus_kernel_file):
  print("Gathering kernels from projects {0}".format(",".join(projects)))
  with open(corpus_kernel_file, "w") as corpus_kernel_file_handle:
    for project in projects:
      project_dir = common.get_project_dir(project)
      #print dot.dot_dirs(project)
      out_dir = dot.dot_dirs(project)[0] # only consider the first one
      project_kernel_file_path = dot.get_kernel_path(project, out_dir)
      
      if os.path.isfile(project_kernel_file_path):
        with open(project_kernel_file_path, "r") as fi: 
            corpus_kernel_file_handle.write(fi.read())
      else:
        print ("No kernel file find for project {0}.\n   {1} is not a file.".format(
          project,
          project_kernel_file_path
          ))
        sys.exit(0)

def generate_project_kernel(project, cluster_json=None):
  """ run graph kernel computation """
  
  project_dir = common.get_project_dir(project)

  out_dir = dot.dot_dirs(project)[0]
  
  kernel_file_path = dot.get_kernel_path(project, out_dir)
  
  if cluster_json:
    print("Using clustering output for node relabeling:")
    graph_kernel_cmd = ['python',
                        common.get_simprog('precompute_kernel.py'),
                        project_dir,
                        kernel_file_path,
                        cluster_json
                        ]
    common.run_cmd(graph_kernel_cmd, True)
  else:
    graph_kernel_cmd = ['python',
                        common.get_simprog('precompute_kernel.py'),
                        project_dir,
                        kernel_file_path
                        ]
    common.run_cmd(graph_kernel_cmd, True)
    
  print("Generated kernel file for {0} in {1}.".format(project, kernel_file_path))

def get_method_map(project_list):
  dot_to_method_map = {}
  for project in project_list:
    for output_dir in [dot.dot_dirs(project)[0]]: # first folder only for now
      method_file = dot.get_method_path(project, output_dir)
      if not os.path.isfile(method_file):
        print ("Cannot find method file for project {0} at {1}".format(project, method_file))
        sys.exit(0)

      with open(method_file, "r") as mf:
        content = mf.readlines()
        for line in content:
          line = line.rstrip()
          items = line.split('\t')
          method_name = items[0]
          method_dot = items[1]
          method_dot_path = dot.get_dot_path(project, output_dir, method_dot)          
          dot_to_method_map[method_dot_path] = method_name
  return dot_to_method_map

def check_similarity(project, result_file, kernel_file, corpus_dot_to_method_map, output_json_file, cluster_json=None, top_k=5):
  """ SUMMARY: use case of the user-driven functionality of PASCALI.
  """

  # fetch various method information from each project in the list
  # output_dir = dot.dot_dirs(project)[0]
  # method_file = dot.get_method_path(project, output_dir)

  # check similarity
  json_result = {}
  sim = Similarity()
  sim.read_graph_kernels(kernel_file)
  iter_num = 3 # number of iteration of the WL-Kernel method
  this_method_map = get_method_map([project])
  with open(result_file, "w") as fo:    
  	for dot_file in this_method_map.keys():
  		dot_method = corpus_dot_to_method_map[dot_file]
  		json_result[dot_method] = []
  		result_program_list_with_score = sim.find_top_k_similar_graphs(dot_file, dot_file, top_k, iter_num, cluster_json)
  		line = dot_file+":\n"
  		for (dt, score) in result_program_list_with_score:
  			line += "{} , {}\n".format(dt, score)
  			if dt not in corpus_dot_to_method_map:
  				print("{0} does not exist.".format(dt))
  				sys.exit(0)
  		tmp_dict = {}
  		tmp_dict[corpus_dot_to_method_map[dt]] = score
  		json_result[dot_method].append(tmp_dict)
  		line += "\n"
  		fo.write(line)
  with open(output_json_file, "w") as jo:
  	jo.write(json.dumps(json_result, indent=4))

def main():
	
	parser = argparse.ArgumentParser()

	parser.add_argument("-c", "--cluster", type=str, help="Path to the input json file that contains the clustering information.")
  	parser.add_argument("-d", "--dir", type=str, required=True, help="Output folder.")
  	parser.add_argument("-p", "--plist", type=str, help="A comma separated list of projects to work with.")
  	parser.add_argument("-k", "--kernel", action="store_true", help="Recompute kernel vectors.")
  	parser.add_argument("-g", "--graph", action="store_true", help="Regenerate graphs.")

  	args = parser.parse_args()

  	common.mkdir(args.dir)

  	# determine which projects to consider
	project_list = common.get_project_list()
	if args.plist:
		arg_projects = args.plist.split(',')
		project_list = [project for project in project_list if project in arg_projects]

	# determine if need to regerenate graphs
	if args.graph:
		for project in project_list:
			generate_graphs(project)

    # determine if need to recompute the kernel vectors
	if args.kernel:
		for project in project_list:
{'fraction_non_alphanumeric': 0.10616256419337701, 'fraction_numerical': 0.003895873915353285, 'mean_word_length': 3.1556291390728477, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4842738', 'n_tokens_mistral': 3673, 'n_tokens_neox': 3530, 'n_words': 843}