<commit_before>from __future__ import (division, absolute_import, print_function,
                        unicode_literals)
from future import standard_library
from future.builtins import *

from brules import Context
from brules.rules import Rule
from brules.helpers.basic import basic_step_set
from brules.helpers.html import html_step_set

from argparse import ArgumentParser
from lxml.html import parse
try:
    from urllib.request import urlopen
except ImportError:
    from urllib2 import urlopen

from importlib import import_module
import json
import logging

log = logging.getLogger(__name__)


def run():
    import sys
    main(sys.argv[1:])


def main(argv):
    parser = ArgumentParser(description='Run some rules on a webpage')
    parser.add_argument('url')
    parser.add_argument('-d', '--dir', default='rules', required=False)
    parser.add_argument('-D', '--debug', default=False, action='store_true')
    parser.add_argument('-v', '--verbose', default=False, action='store_true')
    parser.add_argument('-s', '--extra-stepset', default=None,
                        help="i.e. package.mod:step_set")
    parser.add_argument('-c', '--context-factory', default=None,
                        help="i.e. package.mod:ContextSubclass")
    parser.add_argument('-o', '--output', default='brule_output.json')
    args = parser.parse_args(argv)

    if args.debug:
        logging.basicConfig(level=logging.DEBUG)

    etree = load_url(args.url)
    ctx_class = (load_mod_var(args.context_factory)
                 if args.context_factory else Context)

    context = ctx_class(etree=etree, url=args.url)
    rules = load_rules(args.dir, args.extra_stepset, context)
    run_rules(rules)
    print_results(rules, args.verbose)
    write_results(rules, args.output)


def load_rules(rule_dir, extra_stepset, context):
    base_rule = Rule()
    base_rule.context = context
    if extra_stepset:
        base_rule.add_step_set(load_mod_var(extra_stepset))
    base_rule.add_step_set(basic_step_set)
    base_rule.add_step_set(html_step_set)
    return base_rule.load_directory(rule_dir)


def load_mod_var(var_path):
    mod_name, var_name = var_path.split(':')
    mod = import_module(mod_name)
    return getattr(mod, var_name)


def load_url(url):
    log.debug('Loading %s', url)
    req = urlopen(url)
    if req.geturl() != url:
        log.debug('Redirected to %s', req.geturl())
    return parse(req)


def run_rules(rules):
    for rule in rules:
        rule.run()


def print_results(rules, verbose):
    for rule in rules:
        status = 'Rule: {} ({})'.format(
            rule.metadata.get('name', rule.file_path),
            rule.metadata.get('id', rule.file_path))
        status = red(status) if rule.context.get('fail') else green(status)
        print(status)
        if verbose or rule.context.get('fail'):
            print(id(rule.context))
            keylen = max(len(str(k)) for k in rule.context) + 1
            for k, v in rule.context.items():
                print('  {}: {}'.format(str(k).ljust(keylen), v))<commit_msg>Remove extraneous debug print<commit_after>from __future__ import (division, absolute_import, print_function,
                        unicode_literals)
from future import standard_library
from future.builtins import *

from brules import Context
from brules.rules import Rule
from brules.helpers.basic import basic_step_set
from brules.helpers.html import html_step_set

from argparse import ArgumentParser
from lxml.html import parse
try:
    from urllib.request import urlopen
except ImportError:
    from urllib2 import urlopen

from importlib import import_module
import json
import logging

log = logging.getLogger(__name__)


def run():
    import sys
    main(sys.argv[1:])


def main(argv):
    parser = ArgumentParser(description='Run some rules on a webpage')
    parser.add_argument('url')
    parser.add_argument('-d', '--dir', default='rules', required=False)
    parser.add_argument('-D', '--debug', default=False, action='store_true')
    parser.add_argument('-v', '--verbose', default=False, action='store_true')
    parser.add_argument('-s', '--extra-stepset', default=None,
                        help="i.e. package.mod:step_set")
    parser.add_argument('-c', '--context-factory', default=None,
                        help="i.e. package.mod:ContextSubclass")
    parser.add_argument('-o', '--output', default='brule_output.json')
    args = parser.parse_args(argv)

    if args.debug:
        logging.basicConfig(level=logging.DEBUG)

    etree = load_url(args.url)
    ctx_class = (load_mod_var(args.context_factory)
                 if args.context_factory else Context)

    context = ctx_class(etree=etree, url=args.url)
    rules = load_rules(args.dir, args.extra_stepset, context)
    run_rules(rules)
    print_results(rules, args.verbose)
    write_results(rules, args.output)


def load_rules(rule_dir, extra_stepset, context):
    base_rule = Rule()
    base_rule.context = context
    if extra_stepset:
        base_rule.add_step_set(load_mod_var(extra_stepset))
    base_rule.add_step_set(basic_step_set)
    base_rule.add_step_set(html_step_set)
    return base_rule.load_directory(rule_dir)


def load_mod_var(var_path):
    mod_name, var_name = var_path.split(':')
    mod = import_module(mod_name)
    return getattr(mod, var_name)


def load_url(url):
    log.debug('Loading %s', url)
    req = urlopen(url)
    if req.geturl() != url:
        log.debug('Redirected to %s', req.geturl())
    return parse(req)


def run_rules(rules):
    for rule in rules:
        rule.run()


def print_results(rules, verbose):
    for rule in rules:
        status = 'Rule: {} ({})'.format(
            rule.metadata.get('name', rule.file_path),
            rule.metadata.get('id', rule.file_path))
        status = red(status) if rule.context.get('fail') else green(status)
        print(status)
        if verbose or rule.context.get('fail'):
            keylen = max(len(str(k)) for k in rule.context) + 1
            for k, v in rule.context.items():
                print('  {}: {}'.format(str(k).ljust(keylen), v))
{'fraction_non_alphanumeric': 0.12229862475442044, 'fraction_numerical': 0.0009823182711198428, 'mean_word_length': 3.326487252124646, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2938561', 'n_tokens_mistral': 1963, 'n_tokens_neox': 1930, 'n_words': 427}