<commit_before># Copyright 2016 Capital One Services, LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from __future__ import print_function

from datetime import timedelta, datetime
from functools import wraps
import inspect
import json
import logging
import os
import pprint
import sys
import time

import yaml

from c7n.policy import Policy, load as policy_load
from c7n.reports import report as do_report
from c7n.utils import Bag, dumps
from c7n.manager import resources
from c7n.resources import load_resources
from c7n import schema


log = logging.getLogger('custodian.commands')


def policy_command(f):

    @wraps(f)
    def _load_policies(options):
        load_resources()
        collection = policy_load(options, options.config)
        policies = collection.filter(options.policy_filter)
        
        if options.policy_filter and len(policies) == 0 and len(collection) > 0:
            eprint("Warning: no policies matched the filter ({})".format(options.policy_filter))
            eprint("  Policies:")
            for policy in collection.policies():
                eprint("    - ", policy.name)
            eprint()

        return f(options, policies)

    return _load_policies


def validate(options):
    load_resources()
    if options.config is not None:
        # support the old -c option
        options.configs.append(options.config)
    if len(options.configs) < 1:
        # no configs to test
        # We don't have the parser object, so fake ArgumentParser.error
        eprint('Error: no config files specified')
        sys.exit(1)
    used_policy_names = set()
    schm = schema.generate()
    errors = []
    for config_file in options.configs:
        config_file = os.path.expanduser(config_file)
        if not os.path.exists(config_file):
            raise ValueError("Invalid path for config %r" % config_file)

        options.dryrun = True
        format = config_file.rsplit('.', 1)[-1]
        with open(config_file) as fh:
            if format in ('yml', 'yaml'):
                data = yaml.safe_load(fh.read())
            if format in ('json',):
                data = json.load(fh)

        errors = schema.validate(data, schm)
        conf_policy_names = {p['name'] for p in data.get('policies', ())}
        dupes = conf_policy_names.intersection(used_policy_names)
        if len(dupes) >= 1:
            errors.append(ValueError(
                "Only one policy with a given name allowed, duplicates: %s" % (
                    ", ".join(dupes)
                )
            ))
        used_policy_names = used_policy_names.union(conf_policy_names)
        if not errors:
            null_config = Bag(dryrun=True, log_group=None, cache=None, assume_role="na")
            for p in data.get('policies', ()):
                try:
                    Policy(p, null_config, Bag())
                except Exception as e:
                    msg = "Policy: %s is invalid: %s" % (
                        p.get('name', 'unknown'), e)
                    errors.append(msg)
        if not errors:
            log.info("Configuration valid: {}".format(config_file))
            continue

        log.error("Configuration invalid: {}".format(config_file))
        for e in errors:
            log.error(" %s" % e)
    if errors:
        sys.exit(1)


@policy_command
def access(options, policies):
    permissions = set()
    for p in policies:
        permissions.update(p.get_permissions())
    pprint.pprint(sorted(list(permissions)))


@policy_command
def run(options, policies):
    exit_code = 0
    for policy in policies:
        try:
            policy()
        except Exception:
            exit_code = 2
            if options.debug:
                raise
            log.exception(
                "Error while executing policy %s, continuing" % (
                    policy.name))
    sys.exit(exit_code)


@policy_command
def report(options, policies):
    if len(policies) == 0:
        eprint("Error: No policy supplied")
        sys.exit(1)

    if len(policies) > 1:
        eprint("Error: Report subcommand can only operate on one policy")
        sys.exit(1)
    
    policy = policies.pop()
    d = datetime.now()
    delta = timedelta(days=options.days)
    begin_date = d - delta
    do_report(
        policy, begin_date, options, sys.stdout,
        raw_output_fh=options.raw)


@policy_command
def logs(options, policies):
    if len(policies) == 0:
        eprint("Error: No policy supplied")
        sys.exit(1)

    if len(policies) > 1:
        eprint("Error: Log subcommand can only operate on one policy")
        sys.exit(1)
<commit_msg>cli report - handle policy dir as output directory (#824)<commit_after># Copyright 2016 Capital One Services, LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from __future__ import print_function

from datetime import timedelta, datetime
from functools import wraps
import inspect
import json
import logging
import os
import pprint
import sys
import time

import yaml

from c7n.policy import Policy, load as policy_load
from c7n.reports import report as do_report
from c7n.utils import Bag, dumps
from c7n.manager import resources
from c7n.resources import load_resources
from c7n import schema


log = logging.getLogger('custodian.commands')


def policy_command(f):

    @wraps(f)
    def _load_policies(options):
        load_resources()
        collection = policy_load(options, options.config)
        policies = collection.filter(options.policy_filter)
        
        if options.policy_filter and len(policies) == 0 and len(collection) > 0:
            eprint("Warning: no policies matched the filter ({})".format(options.policy_filter))
            eprint("  Policies:")
            for policy in collection.policies():
                eprint("    - ", policy.name)
            eprint()

        return f(options, policies)

    return _load_policies


def validate(options):
    load_resources()
    if options.config is not None:
        # support the old -c option
        options.configs.append(options.config)
    if len(options.configs) < 1:
        # no configs to test
        # We don't have the parser object, so fake ArgumentParser.error
        eprint('Error: no config files specified')
        sys.exit(1)
    used_policy_names = set()
    schm = schema.generate()
    errors = []
    for config_file in options.configs:
        config_file = os.path.expanduser(config_file)
        if not os.path.exists(config_file):
            raise ValueError("Invalid path for config %r" % config_file)

        options.dryrun = True
        format = config_file.rsplit('.', 1)[-1]
        with open(config_file) as fh:
            if format in ('yml', 'yaml'):
                data = yaml.safe_load(fh.read())
            if format in ('json',):
                data = json.load(fh)

        errors = schema.validate(data, schm)
        conf_policy_names = {p['name'] for p in data.get('policies', ())}
        dupes = conf_policy_names.intersection(used_policy_names)
        if len(dupes) >= 1:
            errors.append(ValueError(
                "Only one policy with a given name allowed, duplicates: %s" % (
                    ", ".join(dupes)
                )
            ))
        used_policy_names = used_policy_names.union(conf_policy_names)
        if not errors:
            null_config = Bag(dryrun=True, log_group=None, cache=None, assume_role="na")
            for p in data.get('policies', ()):
                try:
                    Policy(p, null_config, Bag())
                except Exception as e:
                    msg = "Policy: %s is invalid: %s" % (
                        p.get('name', 'unknown'), e)
                    errors.append(msg)
        if not errors:
            log.info("Configuration valid: {}".format(config_file))
            continue

        log.error("Configuration invalid: {}".format(config_file))
        for e in errors:
            log.error(" %s" % e)
    if errors:
        sys.exit(1)


@policy_command
def access(options, policies):
    permissions = set()
    for p in policies:
        permissions.update(p.get_permissions())
    pprint.pprint(sorted(list(permissions)))


@policy_command
def run(options, policies):
    exit_code = 0
    for policy in policies:
        try:
            policy()
        except Exception:
            exit_code = 2
            if options.debug:
                raise
            log.exception(
                "Error while executing policy %s, continuing" % (
                    policy.name))
    sys.exit(exit_code)


@policy_command
def report(options, policies):
    if len(policies) == 0:
        eprint("Error: No policy supplied")
        sys.exit(1)

    if len(policies) > 1:
        eprint("Error: Report subcommand can only operate on one policy")
        sys.exit(1)
    
    policy = policies.pop()
    odir = options.output_dir.rstrip(os.path.sep)
    if os.path.sep in odir and os.path.basename(odir) == policy.name:
        # policy sub-directory passed - ignore
        options.output_dir = os.path.split(odir)[0]
        # regenerate the execution context based on new path
        policy = Policy(policy.data, options)
    d = datetime.now()
    delta = timedelta(days=options.days)
    begin_date = d - delta
    do_report(
        policy, begin_date, options, sys.stdout,
        raw_output_fh=options.raw)


@policy_command
def logs(options, policies):
    if len(policies) == 0:
        eprint("Error: No policy supplied")
        sys.exit(1)

    if len(policies) > 1:
        eprint("Error: Log subcommand can only operate on one policy")
        sys.exit(1)

{'fraction_non_alphanumeric': 0.09661016949152543, 'fraction_numerical': 0.0064030131826741995, 'mean_word_length': 2.349416587827184, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2796473', 'n_tokens_mistral': 3077, 'n_tokens_neox': 3001, 'n_words': 930}