<commit_before>IN_FILE = 'DecidingForceSchema_Sample/schema1.txt'

TITLE_ID = 'title:'
INSTRUCTIONS_ID = 'instructions:'
GLOSSARY_ID = 'glossary:'
DEPENDENCY_ID = 'if'
DEPENDENCY_TARGET = 'then'

def parse_schema(schema_file=IN_FILE):
    parsed_schema = {}
    with open(schema_file, 'r') as f:
        for line in f:
            raw_line = line.strip()

            # Throw out blank lines
            if not raw_line:
                continue

            # Infer the line type and parse accordingly
            type_id, data = raw_line.split(None, 1)
            if type_id.lower() == TITLE_ID:
                parse_title(data, parsed_schema)
            elif type_id.lower() == INSTRUCTIONS_ID:
                parse_instructions(data, parsed_schema)
            elif type_id.lower() == GLOSSARY_ID:
                parse_glossary(data, parsed_schema)
            elif type_id.lower() == DEPENDENCY_ID:
                parse_dependency(data, parsed_schema)
            else:
                parse_question_entry(type_id, data, parsed_schema)
    return parsed_schema

def parse_title(title, output):
    output['title'] = title

def parse_instructions(instructions, output):
    output['instructions'] = instructions

def parse_glossary(glossary_entry, output):
    if 'glossary' not in output:
        output['glossary'] = {}
    term, definition = glossary_entry.split(':', 1)
    output['glossary'][term.strip()] = definition.strip()

def parse_dependency(dependency, output):
    if 'dependencies' not in output:
        output['dependencies'] = []
    source, target = dependency.split(DEPENDENCY_TARGET, 1)
    output['dependencies'].append((source.strip().strip(','),
                                 target.strip()))

def parse_question_entry(entry_id, data, output):
    type_bits = entry_id.split('.')
    num_bits = len(type_bits)
    if num_bits == 1:
        topic_id = type_bits[0]
        if 'topics' not in output:
            output['topics'] = {}
        output['topics'][topic_id] = {
            'name': data.strip(),
            'questions': {},
        }
    elif num_bits == 2:
        topic_id, question_id = type_bits
        question_id = type_bits[1]
        topic = output['topics'][topic_id]
        question_type, question_text = data.split(None, 1)
        topic['questions'][question_id] = {
            'text': question_text,
            'type': question_type,
            'answers': {}
        }
    else:
        topic_id, question_id, answer_id = type_bits
        question = output['topics'][topic_id]['questions'][question_id]
        question['answers'][answer_id] = {
            'text': data
        }

def print_data(output):
    print "Here's the current parsed data:"
    import pprint; pprint.pprint(output)

if __name__ == '__main__':
    output = parse_schema(IN_FILE)
    print_data(output)
<commit_msg>Better json organization of topics/questions/answers.<commit_after>IN_FILE = 'DecidingForceSchema_Sample/schema1.txt'

TITLE_ID = 'title:'
INSTRUCTIONS_ID = 'instructions:'
GLOSSARY_ID = 'glossary:'
DEPENDENCY_ID = 'if'
DEPENDENCY_TARGET = 'then'

def parse_schema(schema_file=IN_FILE):
    parsed_schema = {}
    with open(schema_file, 'r') as f:
        for line in f:
            raw_line = line.strip()

            # Throw out blank lines
            if not raw_line:
                continue

            # Infer the line type and parse accordingly
            type_id, data = raw_line.split(None, 1)
            if type_id.lower() == TITLE_ID:
                parse_title(data, parsed_schema)
            elif type_id.lower() == INSTRUCTIONS_ID:
                parse_instructions(data, parsed_schema)
            elif type_id.lower() == GLOSSARY_ID:
                parse_glossary(data, parsed_schema)
            elif type_id.lower() == DEPENDENCY_ID:
                parse_dependency(data, parsed_schema)
            else:
                parse_question_entry(type_id, data, parsed_schema)
    return parsed_schema

def parse_title(title, output):
    output['title'] = title

def parse_instructions(instructions, output):
    output['instructions'] = instructions

def parse_glossary(glossary_entry, output):
    if 'glossary' not in output:
        output['glossary'] = {}
    term, definition = glossary_entry.split(':', 1)
    output['glossary'][term.strip()] = definition.strip()

def parse_dependency(dependency, output):
    if 'dependencies' not in output:
        output['dependencies'] = []
    source, target = dependency.split(DEPENDENCY_TARGET, 1)
    output['dependencies'].append((source.strip().strip(','),
                                 target.strip()))

def parse_question_entry(entry_id, data, output):
    type_bits = entry_id.split('.')
    num_bits = len(type_bits)
    if num_bits == 1:
        topic_id = type_bits[0]
        if 'topics' not in output:
            output['topics'] = []
        output['topics'].append({
            'id': topic_id,
            'name': data.strip(),
            'questions': [],
        })
    elif num_bits == 2:
        topic_id, question_id = type_bits
        question_id = type_bits[1]
        topic = [t for t in output['topics'] if t['id'] == topic_id][0]
        question_type, question_text = data.split(None, 1)
        topic['questions'].append({
            'id': question_id,
            'text': question_text,
            'type': question_type,
            'answers': [],
        })
    else:
        topic_id, question_id, answer_id = type_bits
        topic = [t for t in output['topics'] if t['id'] == topic_id][0]
        question = [q for q in topic['questions'] if q['id'] == question_id][0]
        question['answers'].append({
            'id': answer_id,
            'text': data,
        })

def print_data(output):
    print "Here's the current parsed data:"
    import pprint; pprint.pprint(output)

if __name__ == '__main__':
    output = parse_schema(IN_FILE)
    print_data(output)

{'fraction_non_alphanumeric': 0.12026275896917635, 'fraction_numerical': 0.0035371399696816574, 'mean_word_length': 2.4765807962529274, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5245716', 'n_tokens_mistral': 1856, 'n_tokens_neox': 1802, 'n_words': 395}