<commit_before>#! /usr/bin/env python2
import json
import jsonschema
import sys
import os

def get_slice(string_to_slice, slice_index_list):
    # We're slicing a string, so the list should only have the start and stop of the slice.
    assert len(slice_index_list) == 2
    i1 = slice_index_list[0]
    i2 = slice_index_list[1]
    return str(string_to_slice)[i1:i2]

def pretty_print_slice(string_to_slice, slice_index_list):
    assert len(slice_index_list) == 2
    i1 = slice_index_list[0]
    i2 = slice_index_list[1]
    assert i2 > i1
    print " "*i1 + str(string_to_slice)[i1:i2]
    print string_to_slice
    print ' '*i1 + '^' + ' '*(i2-i1-2) + '^'

def find_slice(string, substring):
    # TODO: find all the slices, not just the first one.
    if substring not in string:
        return None
    else:
        start = string.find(substring)
        stop = start + len(substring)
        return [start, stop]


def validate_command(command, uuid_from_filename, oyster_path):

    def assert_custom(assertion, error_string):
        try:
            assert assertion, error_string
        except AssertionError:
            sys.stderr.write("Error in file: "+ oyster_path+'\n')
            traceback.print_exc()
            raise

    def assert_subset(subset, superset):
        assert_custom(subset.issubset(superset), repr(subset)+ "is not a subset of " + repr(superset))

    def assert_in(A, B):
        assert_custom(A in B, repr(A)+ "is not in " + repr(B))

    if 'component-command-info' in command.keys():

        assert_subset(set(command['component-command-info'].keys()), set(command['component-commands']))

        for command_name, info in command['component-command-info'].iteritems():
            for info_key, info_item in info.iteritems():

                if info_key == 'debian':
                    if 'executable-path' in info_item.keys() and info_item['executable-path']:
                        # e.g. `ls` is in `/bin/ls`
                        assert_in(command_name, info_item['executable-path'])

    if uuid_from_filename != command['uuid']:
            sys.stderr.write("Warning in file: "+ oyster_path+'\n')
            sys.stderr.write("Filename does not match UUID. Should be: {}.json\n".format(command['uuid']))

    def validate_invocation(invocation):

        if 'changeable-arguments' in invocation_dict.keys():
            arg_dict = invocation_dict['changeable-arguments']
            if arg_dict:
                for arg, arginfo in arg_dict.iteritems():
                    # Check that the argument actually matches the sliced command.
                    arg_slice = get_slice(invocation_dict['invocation-string'], arginfo['invocation-slice'])
                    try:
                        assert_custom(arg == arg_slice, "arg is:\n'"+arg+"'\nbut slice is:\n'"+arg_slice+"'")
                    except AssertionError:
                        pretty_print_slice(invocation_dict['invocation-string'], arginfo['invocation-slice'])
                        slice_candidate = find_slice(invocation_dict['invocation-string'], arg)
                        if slice_candidate:
                            print "Slice in file:", str(arginfo['invocation-slice'])
                            print "Suggested slice:", str(slice_candidate)
                            pretty_print_slice(invocation_dict['invocation-string'], slice_candidate)
                        raise
                    if 'component-command' in arginfo.keys():
                        assert_in(arginfo['component-command'], command['component-commands'])

        for component_command in command['component-commands']:
            assert_in(component_command, invocation_dict['invocation-string'])

    for invocation_dict in command['invocations']:
        validate_invocation(invocation_dict)

def main(oyster_path, schema_path):

    with open(schema_path) as schema_file:
            try:
                oyster_schema = json.load(schema_file)
            except:
                sys.stderr.write("Invalid JSON in schema: `"+schema_file.name+"'"+'\n')
                raise

    with open(oyster_path) as json_file:
            try:
                oyster = json.load(json_file)
            except:
                sys.stderr.write("Invalid JSON in file: `"+json_file.name+"'"+'\n')
                raise
            try:
                jsonschema.validate(oyster, oyster_schema)
            except jsonschema.exceptions.ValidationError:
                sys.stderr.write(json_file.name+'\n')
                raise
            basename_no_extension = os.path.splitext(os.path.basename(json_file.name))[0]
            validate_command(oyster, basename_no_extension, oyster_path)

if __name__ == '__main__':
    main(sys.argv[1], sys.argv[2])<commit_msg>Make sure there are only two arguments.<commit_after>#! /usr/bin/env python2
import json
import jsonschema
import sys
import os

def get_slice(string_to_slice, slice_index_list):
    # We're slicing a string, so the list should only have the start and stop of the slice.
    assert len(slice_index_list) == 2
    i1 = slice_index_list[0]
    i2 = slice_index_list[1]
    return str(string_to_slice)[i1:i2]

def pretty_print_slice(string_to_slice, slice_index_list):
    assert len(slice_index_list) == 2
    i1 = slice_index_list[0]
    i2 = slice_index_list[1]
    assert i2 > i1
    print " "*i1 + str(string_to_slice)[i1:i2]
    print string_to_slice
    print ' '*i1 + '^' + ' '*(i2-i1-2) + '^'

def find_slice(string, substring):
    # TODO: find all the slices, not just the first one.
    if substring not in string:
        return None
    else:
        start = string.find(substring)
        stop = start + len(substring)
        return [start, stop]


def validate_command(command, uuid_from_filename, oyster_path):

    def assert_custom(assertion, error_string):
        try:
            assert assertion, error_string
        except AssertionError:
            sys.stderr.write("Error in file: "+ oyster_path+'\n')
            traceback.print_exc()
            raise

    def assert_subset(subset, superset):
        assert_custom(subset.issubset(superset), repr(subset)+ "is not a subset of " + repr(superset))

    def assert_in(A, B):
        assert_custom(A in B, repr(A)+ "is not in " + repr(B))

    if 'component-command-info' in command.keys():

        assert_subset(set(command['component-command-info'].keys()), set(command['component-commands']))

        for command_name, info in command['component-command-info'].iteritems():
            for info_key, info_item in info.iteritems():

                if info_key == 'debian':
                    if 'executable-path' in info_item.keys() and info_item['executable-path']:
                        # e.g. `ls` is in `/bin/ls`
                        assert_in(command_name, info_item['executable-path'])

    if uuid_from_filename != command['uuid']:
            sys.stderr.write("Warning in file: "+ oyster_path+'\n')
            sys.stderr.write("Filename does not match UUID. Should be: {}.json\n".format(command['uuid']))

    def validate_invocation(invocation):

        if 'changeable-arguments' in invocation_dict.keys():
            arg_dict = invocation_dict['changeable-arguments']
            if arg_dict:
                for arg, arginfo in arg_dict.iteritems():
                    # Check that the argument actually matches the sliced command.
                    arg_slice = get_slice(invocation_dict['invocation-string'], arginfo['invocation-slice'])
                    try:
                        assert_custom(arg == arg_slice, "arg is:\n'"+arg+"'\nbut slice is:\n'"+arg_slice+"'")
                    except AssertionError:
                        pretty_print_slice(invocation_dict['invocation-string'], arginfo['invocation-slice'])
                        slice_candidate = find_slice(invocation_dict['invocation-string'], arg)
                        if slice_candidate:
                            print "Slice in file:", str(arginfo['invocation-slice'])
                            print "Suggested slice:", str(slice_candidate)
                            pretty_print_slice(invocation_dict['invocation-string'], slice_candidate)
                        raise
                    if 'component-command' in arginfo.keys():
                        assert_in(arginfo['component-command'], command['component-commands'])

        for component_command in command['component-commands']:
            assert_in(component_command, invocation_dict['invocation-string'])

    for invocation_dict in command['invocations']:
        validate_invocation(invocation_dict)

def main(oyster_path, schema_path):

    with open(schema_path) as schema_file:
            try:
                oyster_schema = json.load(schema_file)
            except:
                sys.stderr.write("Invalid JSON in schema: `"+schema_file.name+"'"+'\n')
                raise

    with open(oyster_path) as json_file:
            try:
                oyster = json.load(json_file)
            except:
                sys.stderr.write("Invalid JSON in file: `"+json_file.name+"'"+'\n')
                raise
            try:
                jsonschema.validate(oyster, oyster_schema)
            except jsonschema.exceptions.ValidationError:
                sys.stderr.write(json_file.name+'\n')
                raise
            basename_no_extension = os.path.splitext(os.path.basename(json_file.name))[0]
            validate_command(oyster, basename_no_extension, oyster_path)

if __name__ == '__main__':
    num_args = len(sys.argv) - 1
    if num_args != 2:
        sys.stderr.write("Usage: python "+sys.argv[0]+" cmd-oyster.json schema.json"+'\n')
        sys.exit(1)
    main(sys.argv[1], sys.argv[2])
{'fraction_non_alphanumeric': 0.11201065792170527, 'fraction_numerical': 0.005533920885427342, 'mean_word_length': 2.3341305090536384, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4849397', 'n_tokens_mistral': 2892, 'n_tokens_neox': 2787, 'n_words': 613}