<commit_before># Copyright 2017 Capital One Services, LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Run a custodian policy across an organization's accounts
"""

from collections import Counter
import logging
import os
import multiprocessing
import time
import subprocess
import sys

from concurrent.futures import (
    ProcessPoolExecutor,
    as_completed)
import yaml

import boto3
from botocore.compat import OrderedDict
from botocore.exceptions import ClientError
import click
import jsonschema

from c7n.credentials import assumed_session, SessionFactory
from c7n.executor import MainThreadExecutor
from c7n.config import Config
from c7n.policy import PolicyCollection
from c7n.reports.csvout import Formatter, fs_record_set
from c7n.resources import load_resources
from c7n.manager import resources as resource_registry
from c7n.utils import CONN_CACHE, dumps

from c7n_org.utils import environ, account_tags
from c7n.utils import UnicodeWriter

log = logging.getLogger('c7n_org')

# On OSX High Sierra Workaround
# https://github.com/ansible/ansible/issues/32499
if sys.platform == 'darwin':
    os.environ['OBJC_DISABLE_INITIALIZE_FORK_SAFETY'] = 'YES'

WORKER_COUNT = int(
    os.environ.get('C7N_ORG_PARALLEL', multiprocessing.cpu_count() * 4))


CONFIG_SCHEMA = {
    '$schema': 'http://json-schema.org/schema#',
    'id': 'http://schema.cloudcustodian.io/v0/orgrunner.json',
    'definitions': {
        'account': {
            'type': 'object',
            'additionalProperties': True,
            'anyOf': [
                {'required': ['role', 'account_id']},
                {'required': ['profile', 'account_id']}
            ],
            'properties': {
                'name': {'type': 'string'},
                'email': {'type': 'string'},
                'account_id': {'type': 'string'},
                'profile': {'type': 'string', 'minLength': 3},
                'tags': {'type': 'array', 'items': {'type': 'string'}},
                'regions': {'type': 'array', 'items': {'type': 'string'}},
                'role': {'oneOf': [
                    {'type': 'array', 'items': {'type': 'string'}},
                    {'type': 'string', 'minLength': 3}]},
                'external_id': {'type': 'string'},
                'vars': {'type': 'object'},
            }
        },
        'subscription': {
            'type': 'object',
            'additionalProperties': False,
            'required': ['subscription_id'],
            'properties': {
                'subscription_id': {'type': 'string'},
                'tags': {'type': 'array', 'items': {'type': 'string'}},
                'name': {'type': 'string'},
                'vars': {'type': 'object'},
            }
        },
        'project': {
            'type': 'object',
            'additionalProperties': False,
            'required': ['project_id'],
            'properties': {
                'project_id': {'type': 'string'},
                'tags': {'type': 'array', 'items': {'type': 'string'}},
                'name': {'type': 'string'},
                'vars': {'type': 'object'},
            }
        },
    },
    'type': 'object',
    'additionalProperties': False,
    'oneOf': [
        {'required': ['accounts']},
        {'required': ['projects']},
        {'required': ['subscriptions']}
    ],
    'properties': {
        'vars': {'type': 'object'},
        'accounts': {
            'type': 'array',
            'items': {'$ref': '#/definitions/account'}
        },
        'subscriptions': {
            'type': 'array',
            'items': {'$ref': '#/definitions/subscription'}
        },
        'projects': {
            'type': 'array',
            'items': {'$ref': '#/definitions/project'}
        }
    }
}


@click.group()
def cli():
    """custodian organization multi-account runner."""


class LogFilter(object):
    """We want to keep the main c7n-org cli output to be readable.

    We previously did so via squelching custodian's log output via
    level filter on the logger, however doing that meant that log
    outputs stored to output locations were also squelched.

    We effectively want differential handling at the top level logger
    stream handler, ie. we want `custodian` log messages to propagate
    to the root logger based on level, but we also want them to go the
    custodian logger's directly attached handlers on debug level.
    """

    def filter(self, r):
        if not r.name.startswith('custodian'):
            return 1
        elif r.levelno >= logging.WARNING:
            return 1
        return 0


def init(config, use, debug, verbose, accounts, tags, policies, resource=None, policy_tags=()):
    level = verbose and logging.DEBUG or logging.INFO
    logging.basicConfig(
        level=level,
        format="%(asctime)s: %(name)s:%(levelname)s %(message)s")

    logging.getLogger('botocore').setLevel(logging.ERROR)
    logging.getLogger('s3transfer').setLevel(logging.WARNING)
    logging.getLogger('custodian.s3').setLevel(logging.ERROR)
    logging.getLogger('urllib3').setLevel(logging.WARNING)

    # Filter out custodian log messages on console output if not
    # at warning level or higher, see LogFilter docs and #2674
    for h in logging.getLogger().handlers:
        if isinstance(h, logging.StreamHandler):
            h.addFilter(LogFilter())

    with open(config) as fh:
        accounts_config = yaml.safe_load(fh.read())<commit_msg>tools/c7n-org - py3 compat -  cli.py open policy files in binary mode (#3732) (#3733)<commit_after># Copyright 2017 Capital One Services, LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Run a custodian policy across an organization's accounts
"""

from collections import Counter
import logging
import os
import multiprocessing
import time
import subprocess
import sys

from concurrent.futures import (
    ProcessPoolExecutor,
    as_completed)
import yaml

import boto3
from botocore.compat import OrderedDict
from botocore.exceptions import ClientError
import click
import jsonschema

from c7n.credentials import assumed_session, SessionFactory
from c7n.executor import MainThreadExecutor
from c7n.config import Config
from c7n.policy import PolicyCollection
from c7n.reports.csvout import Formatter, fs_record_set
from c7n.resources import load_resources
from c7n.manager import resources as resource_registry
from c7n.utils import CONN_CACHE, dumps

from c7n_org.utils import environ, account_tags
from c7n.utils import UnicodeWriter

log = logging.getLogger('c7n_org')

# On OSX High Sierra Workaround
# https://github.com/ansible/ansible/issues/32499
if sys.platform == 'darwin':
    os.environ['OBJC_DISABLE_INITIALIZE_FORK_SAFETY'] = 'YES'

WORKER_COUNT = int(
    os.environ.get('C7N_ORG_PARALLEL', multiprocessing.cpu_count() * 4))


CONFIG_SCHEMA = {
    '$schema': 'http://json-schema.org/schema#',
    'id': 'http://schema.cloudcustodian.io/v0/orgrunner.json',
    'definitions': {
        'account': {
            'type': 'object',
            'additionalProperties': True,
            'anyOf': [
                {'required': ['role', 'account_id']},
                {'required': ['profile', 'account_id']}
            ],
            'properties': {
                'name': {'type': 'string'},
                'email': {'type': 'string'},
                'account_id': {'type': 'string'},
                'profile': {'type': 'string', 'minLength': 3},
                'tags': {'type': 'array', 'items': {'type': 'string'}},
                'regions': {'type': 'array', 'items': {'type': 'string'}},
                'role': {'oneOf': [
                    {'type': 'array', 'items': {'type': 'string'}},
                    {'type': 'string', 'minLength': 3}]},
                'external_id': {'type': 'string'},
                'vars': {'type': 'object'},
            }
        },
        'subscription': {
            'type': 'object',
            'additionalProperties': False,
            'required': ['subscription_id'],
            'properties': {
                'subscription_id': {'type': 'string'},
                'tags': {'type': 'array', 'items': {'type': 'string'}},
                'name': {'type': 'string'},
                'vars': {'type': 'object'},
            }
        },
        'project': {
            'type': 'object',
            'additionalProperties': False,
            'required': ['project_id'],
            'properties': {
                'project_id': {'type': 'string'},
                'tags': {'type': 'array', 'items': {'type': 'string'}},
                'name': {'type': 'string'},
                'vars': {'type': 'object'},
            }
        },
    },
    'type': 'object',
    'additionalProperties': False,
    'oneOf': [
        {'required': ['accounts']},
        {'required': ['projects']},
        {'required': ['subscriptions']}
    ],
    'properties': {
        'vars': {'type': 'object'},
        'accounts': {
            'type': 'array',
            'items': {'$ref': '#/definitions/account'}
        },
        'subscriptions': {
            'type': 'array',
            'items': {'$ref': '#/definitions/subscription'}
        },
        'projects': {
            'type': 'array',
            'items': {'$ref': '#/definitions/project'}
        }
    }
}


@click.group()
def cli():
    """custodian organization multi-account runner."""


class LogFilter(object):
    """We want to keep the main c7n-org cli output to be readable.

    We previously did so via squelching custodian's log output via
    level filter on the logger, however doing that meant that log
    outputs stored to output locations were also squelched.

    We effectively want differential handling at the top level logger
    stream handler, ie. we want `custodian` log messages to propagate
    to the root logger based on level, but we also want them to go the
    custodian logger's directly attached handlers on debug level.
    """

    def filter(self, r):
        if not r.name.startswith('custodian'):
            return 1
        elif r.levelno >= logging.WARNING:
            return 1
        return 0


def init(config, use, debug, verbose, accounts, tags, policies, resource=None, policy_tags=()):
    level = verbose and logging.DEBUG or logging.INFO
    logging.basicConfig(
        level=level,
        format="%(asctime)s: %(name)s:%(levelname)s %(message)s")

    logging.getLogger('botocore').setLevel(logging.ERROR)
    logging.getLogger('s3transfer').setLevel(logging.WARNING)
    logging.getLogger('custodian.s3').setLevel(logging.ERROR)
    logging.getLogger('urllib3').setLevel(logging.WARNING)

    # Filter out custodian log messages on console output if not
    # at warning level or higher, see LogFilter docs and #2674
    for h in logging.getLogger().handlers:
        if isinstance(h, logging.StreamHandler):
            h.addFilter(LogFilter())

    with open(config, 'rb') as fh:
        accounts_config = yaml.safe_load(fh.read())
{'fraction_non_alphanumeric': 0.13864701942397856, 'fraction_numerical': 0.007702612190221032, 'mean_word_length': 2.620794180054562, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 2, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7031688', 'n_tokens_mistral': 3476, 'n_tokens_neox': 3337, 'n_words': 1015}