<commit_before>"""
Searches the given path for JSON files, and validates their contents.

Optionally, replaces valid JSON files with their pretty-printed
counterparts.
"""

import errno
import json
import logging
import optparse
import os
import re 


# Configure logging
logging.basicConfig(format='%(levelname)s: %(message)s')
ROOT_LOGGER = logging.getLogger("")
ROOT_LOGGER.setLevel(logging.WARNING)
LOGGER = logging.getLogger(__name__)

# Configure commandlineability
parser = optparse.OptionParser()
parser.add_option('-p', type="string", default='.',
    help='the path to search for JSON files', dest='path')
parser.add_option('-r', type="string", default='.json$',
    help='the regular expression to match filenames against ' \
        '(not absolute paths)', dest='regexp')
(args, _) = parser.parse_args()


def main():
    files = find_matching_files(args.path, args.regexp)
    results = True
    for path in files:
        results &= validate_json(path)
    # Invert our test results to produce a status code
    exit(not results)


def validate_json(path):
    """Open a file and validate it's contents as JSON"""
    try:
        LOGGER.info("Validating %s" % (path))
        contents = read_file(path)
        if contents is False:
            logging.warning('Insufficient permissions to open: %s' % path)
            return False
    except:
        LOGGER.warning('Unable to open: %s' % path)
        return False

    #knock off comments<commit_msg>Updating to show that we did something.<commit_after>"""
Searches the given path for JSON files, and validates their contents.

Optionally, replaces valid JSON files with their pretty-printed
counterparts.
"""

import errno
import json
import logging
import optparse
import os
import re 


# Configure logging
logging.basicConfig(format='%(levelname)s: %(message)s')
ROOT_LOGGER = logging.getLogger("")
ROOT_LOGGER.setLevel(logging.WARNING)
LOGGER = logging.getLogger(__name__)

# Configure commandlineability
parser = optparse.OptionParser()
parser.add_option('-p', type="string", default='.',
    help='the path to search for JSON files', dest='path')
parser.add_option('-r', type="string", default='.json$',
    help='the regular expression to match filenames against ' \
        '(not absolute paths)', dest='regexp')
(args, _) = parser.parse_args()


def main():
    files = find_matching_files(args.path, args.regexp)
    results = True
    print("Validating %s json files (found using regex %s in path %s)" % (len(files), args.regexp, args.path))
    for path in files:
        results &= validate_json(path)
    # Invert our test results to produce a status code
    exit(not results)


def validate_json(path):
    """Open a file and validate it's contents as JSON"""
    try:
        LOGGER.info("Validating %s" % (path))
        contents = read_file(path)
        if contents is False:
            logging.warning('Insufficient permissions to open: %s' % path)
            return False
    except:
        LOGGER.warning('Unable to open: %s' % path)
        return False

    #knock off comments
{'fraction_non_alphanumeric': 0.11414634146341464, 'fraction_numerical': 0.0, 'mean_word_length': 3.7250384024577574, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1423167', 'n_tokens_mistral': 904, 'n_tokens_neox': 887, 'n_words': 293}