<commit_before># -*- encoding: utf-8 -*-
from __future__ import (unicode_literals, absolute_import,
                        division, print_function)
import argparse
import sys
from . import version
from .validator import OParlJson
from .crawler import Crawler


def main():
    parser = argparse.ArgumentParser(
        description='Validates an OParl system or OParl documents.')
    parser.add_argument(
        'url', metavar='URL', nargs='?', default=None,
        help='URL of the system to test; omit to read JSON from stdin')
    parser.add_argument(
        '-t', '--type', metavar='type', dest='types',
        default=None, action='append',
        help='Set the document types to be checked '
             '(can be used multiple times)')
    parser.add_argument(
        '-r', '--recursive',
        action='store_true', help='Test the whole system recursively')
    parser.add_argument(
        '-n', '--max-documents', metavar='number', type=int, default=None,
        help='Set max number of documents to test per document type')
    parser.add_argument(
        '-c', '--compact', action='store_true',
        help='Gives a compact error report')
    parser.add_argument(
        '-v', '--oparl-version', metavar='version',
        help=('Set the OParl version. '
              'Currently “1” is the only valid option.'))
    parser.add_argument(
        '-V', '--version', action='version',
        version='%(prog)s ' + version.__version__)
    args = parser.parse_args()
    if not args.url:
        for error in OParlJson(sys.stdin.read()).validate():
            print('ERROR: %s' % error.message)
    else:
        crawler = Crawler(seed_url=args.url, max_documents=args.max_documents,
                          type_whitelist=args.types, recursive=args.recursive)
        crawler.run()<commit_msg>cli: simple display for the errors<commit_after># -*- encoding: utf-8 -*-
from __future__ import (unicode_literals, absolute_import,
                        division, print_function)
import argparse
import sys
from . import version
from .validator import OParlJson
from .crawler import Crawler


def main():
    parser = argparse.ArgumentParser(
        description='Validates an OParl system or OParl documents.')
    parser.add_argument(
        'url', metavar='URL', nargs='?', default=None,
        help='URL of the system to test; omit to read JSON from stdin')
    parser.add_argument(
        '-t', '--type', metavar='type', dest='types',
        default=None, action='append',
        help='Set the document types to be checked '
             '(can be used multiple times)')
    parser.add_argument(
        '-r', '--recursive',
        action='store_true', help='Test the whole system recursively')
    parser.add_argument(
        '-n', '--max-documents', metavar='number', type=int, default=None,
        help='Set max number of documents to test per document type')
    parser.add_argument(
        '-c', '--compact', action='store_true',
        help='Gives a compact error report')
    parser.add_argument(
        '-v', '--oparl-version', metavar='version',
        help=('Set the OParl version. '
              'Currently “1” is the only valid option.'))
    parser.add_argument(
        '-V', '--version', action='version',
        version='%(prog)s ' + version.__version__)
    args = parser.parse_args()
    if not args.url:
        for error in OParlJson(sys.stdin.read()).validate():
            print('ERROR: %s' % error.message)
    else:
        crawler = Crawler(seed_url=args.url, max_documents=args.max_documents,
                          type_whitelist=args.types, recursive=args.recursive)
        for url, error in crawler.run():
            print('ERROR in %s: %s' % (url, error.message))
{'fraction_non_alphanumeric': 0.13288770053475935, 'fraction_numerical': 0.0010695187165775401, 'mean_word_length': 2.674852652259332, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1267012', 'n_tokens_mistral': 1096, 'n_tokens_neox': 1080, 'n_words': 306}