<commit_before>#!/usr/bin/python

#
# Convert vspec file to CSV
#

import sys
import vspec
import json
import getopt
import csv

def usage():
    print "Usage:", sys.argv[0], "[-I include_dir] ... [-i prefix:id_file:start_id] vspec_file csv_file"
    print "  -I include_dir              Add include directory to search for included vspec"
    print "                              files. Can be used multiple timees."
    print
    print "  -i prefix:id_file:start_id  Add include directory to search for included vspec"
    print "                              files. Can be used multiple timees."
    print
    print " vspec_file                   The vehicle specification file to parse."
    print " csv_file                    The file to output the CSV data to."
    sys.exit(255)

def format_data(json_data):
    Id = '""'
    Type = '""'
    Unit = '""'
    Min = '""'
    Max = '""'
    Desc = '""'
    if (json_data.has_key('id')):
        Id = '"' + str(json_data['id']) + '"'
    if (json_data.has_key('type')):
        Type = '"' + json_data['type'] + '"'
    if (json_data.has_key('unit')):
        Unit = '"' + json_data['unit'] + '"'
    if (json_data.has_key('min')):
        Min = '"' + str(json_data['min']) + '"'
    if (json_data.has_key('max')):
        Max = '"' + str(json_data['max']) + '"'
    if (json_data.has_key('description')):
        Desc = '"' + json_data['description'] + '"'
    return Id + ","+ Type + ","+ Unit + ","+ Min + ","+ Max + ","+ Desc

def json2csv(json_data, file_out, parent_signal):
    for k in json_data.keys():
        if (len(parent_signal) > 0):
            signal = parent_signal + "." + k
        else:
            signal = k

        if (json_data[k]['type'] == 'branch'):
            file_out.write(signal + ',' + format_data(json_data[k]) + '\n')
            json2csv(json_data[k]['children'], file_out, signal)
        else:<commit_msg>Added 'enum' handling in csv conversion tool 'enum' had been ignored in vspec2csv.py<commit_after>#!/usr/bin/python

#
# Convert vspec file to CSV
#

import sys
import vspec
import json
import getopt
import csv

def usage():
    print "Usage:", sys.argv[0], "[-I include_dir] ... [-i prefix:id_file:start_id] vspec_file csv_file"
    print "  -I include_dir              Add include directory to search for included vspec"
    print "                              files. Can be used multiple timees."
    print
    print "  -i prefix:id_file:start_id  Add include directory to search for included vspec"
    print "                              files. Can be used multiple timees."
    print
    print " vspec_file                   The vehicle specification file to parse."
    print " csv_file                    The file to output the CSV data to."
    sys.exit(255)

def format_data(json_data):
    Id = '""'
    Type = '""'
    Unit = '""'
    Min = '""'
    Max = '""'
    Desc = '""'
    Enum = '""'
    if (json_data.has_key('id')):
        Id = '"' + str(json_data['id']) + '"'
    if (json_data.has_key('type')):
        Type = '"' + json_data['type'] + '"'
    if (json_data.has_key('unit')):
        Unit = '"' + json_data['unit'] + '"'
    if (json_data.has_key('min')):
        Min = '"' + str(json_data['min']) + '"'
    if (json_data.has_key('max')):
        Max = '"' + str(json_data['max']) + '"'
    if (json_data.has_key('description')):
        Desc = '"' + json_data['description'] + '"'
    if (json_data.has_key('enum')):
        Enum = '"' + ' / '.join(json_data['enum']) + '"'
    return Id + ","+ Type + ","+ Unit + ","+ Min + ","+ Max + ","+ Desc + ","+ Enum

def json2csv(json_data, file_out, parent_signal):
    for k in json_data.keys():
        if (len(parent_signal) > 0):
            signal = parent_signal + "." + k
        else:
            signal = k

        if (json_data[k]['type'] == 'branch'):
            file_out.write(signal + ',' + format_data(json_data[k]) + '\n')
            json2csv(json_data[k]['children'], file_out, signal)
        else:
{'fraction_non_alphanumeric': 0.1675025075225677, 'fraction_numerical': 0.003761283851554664, 'mean_word_length': 2.237824675324675, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2713419', 'n_tokens_mistral': 1328, 'n_tokens_neox': 1300, 'n_words': 317}