<commit_before>import sys, os
import subprocess32 as subprocess
import json
import glob2 as glob
# assume pa2checker is under the same dir of this script
import common, pa2checker, frontend_pa_inference

MAP_WORKING_DIR = os.path.dirname(os.path.realpath(__file__))

JAIF_FILE_ONTOLOGY_HEADER ="\
package ontology.qual:\n\
  annotation @Ontology:\n\
    enum ontology.qual.OntologyValue[] values\n"

def main(json_file):
    with open(json_file) as data_file:
        data = json.load(data_file)

    mappings = data["mappings"]
    ontology_set = set()
    project_list = common.get_project_list()

    for mapping in mappings:
        ontology_set.add(mapping['label'][0])

    convertToOntologyValue(ontology_set)

    corpus_jaif_file = createSingleJaifFile4Corpus(mappings)

    for project in project_list:
        runInsertAnnoToProject(project, corpus_jaif_file)

    for project in project_list:
        frontend_pa_inference.run_inference(project)

def runInsertAnnoToProject(project, jaif_file):
    """ Insert annotation info in the ${jaif_file} to ${project}.
    """
    project_dir = common.get_project_dir(project)
    with common.cd(project_dir):
        common.setup_checker_framework_env()
        insert_cmd = ['insert-annotations-to-source', '-i', jaif_file]
        # using glob2.glob to recursive get java files under project dir
        java_files = glob.glob('{}/**/*.java'.format(project_dir))
        insert_cmd.extend(java_files)
        common.run_cmd(insert_cmd, print_output=True)

def createSingleJaifFile4Corpus(mappings):<commit_msg>only generate jaif file. do not insert and infer.<commit_after>import sys, os
import subprocess32 as subprocess
import json
import glob2 as glob
# assume pa2checker is under the same dir of this script
import common, pa2checker, frontend_pa_inference

MAP_WORKING_DIR = os.path.dirname(os.path.realpath(__file__))

JAIF_FILE_ONTOLOGY_HEADER ="\
package ontology.qual:\n\
  annotation @Ontology:\n\
    enum ontology.qual.OntologyValue[] values\n"

def main(json_file):
    with open(json_file) as data_file:
        data = json.load(data_file)

    mappings = data["mappings"]
    ontology_set = set()
    project_list = common.get_project_list()

    for mapping in mappings:
        ontology_set.add(mapping['label'][0])

    convertToOntologyValue(ontology_set)

    corpus_jaif_file = createSingleJaifFile4Corpus(mappings)

    # for project in project_list:
    #     runInsertAnnoToProject(project, corpus_jaif_file)

    # for project in project_list:
    #     frontend_pa_inference.run_inference(project)

def runInsertAnnoToProject(project, jaif_file):
    """ Insert annotation info in the ${jaif_file} to ${project}.
    """
    project_dir = common.get_project_dir(project)
    with common.cd(project_dir):
        common.setup_checker_framework_env()
        insert_cmd = ['insert-annotations-to-source', '-i', jaif_file]
        # using glob2.glob to recursive get java files under project dir
        java_files = glob.glob('{}/**/*.java'.format(project_dir))
        insert_cmd.extend(java_files)
        common.run_cmd(insert_cmd, print_output=True)

def createSingleJaifFile4Corpus(mappings):
{'fraction_non_alphanumeric': 0.09565764631843927, 'fraction_numerical': 0.005663939584644431, 'mean_word_length': 4.135702746365105, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6672532', 'n_tokens_mistral': 1045, 'n_tokens_neox': 1010, 'n_words': 221}