<commit_before># Copyright 2017 Google Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""GCP Resource scanner.

Usage:

  $ forseti_scanner --rules <rules path> \\
      --output_path <output path (optional)> \\
      --db_host <Cloud SQL database hostname/IP> \\
      --db_user <Cloud SQL database user> \\
      --db_name <Cloud SQL database name (required)> \\
      --sendgrid_api_key <API key to auth SendGrid email service> \\
      --email_sender <email address of the email sender> \\
      --email_recipient <email address of the email recipient>
"""

import itertools
import os
import shutil
import sys

from datetime import datetime

import gflags as flags

from google.apputils import app
from google.cloud.security.common.data_access import csv_writer
from google.cloud.security.common.data_access import dao
from google.cloud.security.common.data_access import organization_dao
from google.cloud.security.common.data_access import project_dao
from google.cloud.security.common.data_access import violation_dao
from google.cloud.security.common.data_access import errors as db_errors
from google.cloud.security.common.gcp_type.resource import ResourceType
from google.cloud.security.common.gcp_type.resource_util import ResourceUtil
from google.cloud.security.common.util import log_util
from google.cloud.security.common.util.email_util import EmailUtil
from google.cloud.security.scanner.audit.iam_rules_engine import IamRulesEngine

# Setup flags
FLAGS = flags.FLAGS

# Format: flags.DEFINE_<type>(flag_name, default_value, help_text)
# Example:
# https://github.com/google/python-gflags/blob/master/examples/validator.py
flags.DEFINE_string('rules', None,
                    ('Path to rules file (yaml/json). '
                     'If GCS object, include full path, e.g. '
                     ' "gs://<bucketname>/path/to/file".'))

flags.DEFINE_string('group_rules', None,
                    ('Path to rules file (yaml/json). '
                     'If GCS object, include full path, e.g. '
                     ' "gs://<bucketname>/path/to/file".'))

flags.DEFINE_string('output_path', None,
                    ('Output path (do not include filename). If GCS location, '
                     'the format of the path should be '
                     '"gs://bucket-name/path/for/output".'))

LOGGER = log_util.get_logger(__name__)
SCANNER_OUTPUT_CSV_FMT = 'scanner_output.{}.csv'
OUTPUT_TIMESTAMP_FMT = '%Y%m%dT%H%M%SZ'


def main(_):
    """Run the scanner."""

    LOGGER.info('Initializing the rules engine:\nUsing rules: %s', FLAGS.rules)

    if not FLAGS.rules:
        LOGGER.warn(('Provide a rules file. '
                     'Use "forseti_scanner --helpfull" for help.'))
        sys.exit(1)

    rules_engine = IamRulesEngine(rules_file_path=FLAGS.rules)
    rules_engine.build_rule_book()

    snapshot_timestamp = _get_timestamp()
    if not snapshot_timestamp:
        LOGGER.warn('No snapshot timestamp found. Exiting.')
        sys.exit()

    # TODO: make this generic
    org_policies = _get_org_policies(snapshot_timestamp)
    project_policies = _get_project_policies(snapshot_timestamp)

    if not org_policies and not project_policies:
        LOGGER.warn('No policies found. Exiting.')
        sys.exit()

    all_violations = _find_violations(
        itertools.chain(
            org_policies.iteritems(),
            project_policies.iteritems()),
        rules_engine)

    # If there are violations, send results.
    if all_violations:
        resource_counts = {
            ResourceType.ORGANIZATION: len(org_policies),
            ResourceType.PROJECT: len(project_policies),
        }
        _output_results(all_violations,
                        snapshot_timestamp,
                        resource_counts=resource_counts)

    LOGGER.info('Done!')

def _find_violations(policies, rules_engine):
    """Find violations in the policies.

    Args:
        policies: The list of policies to find violations in.
        rules_engine: The rules engine to run.

    Returns:
        A list of violations.
    """

    all_violations = []
    LOGGER.info('Finding policy violations...')
    for (resource, policy) in policies:
        LOGGER.debug('%s => %s', resource, policy)
        violations = rules_engine.find_policy_violations(
            resource, policy)
        LOGGER.debug(violations)
        all_violations.extend(violations)
    return all_violations

def _get_output_filename(now_utc):
    """Create the output filename.

    Args:
        now_utc: The datetime now in UTC.

    Returns:
        The output filename for the csv, formatted with the now_utc timestamp.
    """<commit_msg>Improvement on loading rules engines more dynamically. This requires that rules engines have a build_rule_book() function.<commit_after># Copyright 2017 Google Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""GCP Resource scanner.

Usage:

  $ forseti_scanner --rules <rules path> \\
      --output_path <output path (optional)> \\
      --db_host <Cloud SQL database hostname/IP> \\
      --db_user <Cloud SQL database user> \\
      --db_name <Cloud SQL database name (required)> \\
      --sendgrid_api_key <API key to auth SendGrid email service> \\
      --email_sender <email address of the email sender> \\
      --email_recipient <email address of the email recipient>
"""

import itertools
import os
import shutil
import sys
import glob
import ast
import imp

from datetime import datetime

import gflags as flags

from google.apputils import app
from google.cloud.security.common.data_access import csv_writer
from google.cloud.security.common.data_access import dao
from google.cloud.security.common.data_access import organization_dao
from google.cloud.security.common.data_access import project_dao
from google.cloud.security.common.data_access import violation_dao
from google.cloud.security.common.data_access import errors as db_errors
from google.cloud.security.common.gcp_type.resource import ResourceType
from google.cloud.security.common.gcp_type.resource_util import ResourceUtil
from google.cloud.security.common.util import log_util
from google.cloud.security.common.util.email_util import EmailUtil
from google.cloud.security.scanner.audit.iam_rules_engine import IamRulesEngine

# Setup flags
FLAGS = flags.FLAGS

# Format: flags.DEFINE_<type>(flag_name, default_value, help_text)
# Example:
# https://github.com/google/python-gflags/blob/master/examples/validator.py
flags.DEFINE_string('rules', None,
                    ('Path to rules file (yaml/json). '
                     'If GCS object, include full path, e.g. '
                     ' "gs://<bucketname>/path/to/file".'))

flags.DEFINE_string('group_rules', None,
                    ('Path to rules file (yaml/json). '
                     'If GCS object, include full path, e.g. '
                     ' "gs://<bucketname>/path/to/file".'))

flags.DEFINE_string('output_path', None,
                    ('Output path (do not include filename). If GCS location, '
                     'the format of the path should be '
                     '"gs://bucket-name/path/for/output".'))

flags.DEFINE_bool('list_engines', False, 'List all rule engines')

flags.DEFINE_string('use_engine', None, 'Which engine to use')

LOGGER = log_util.get_logger(__name__)
SCANNER_OUTPUT_CSV_FMT = 'scanner_output.{}.csv'
OUTPUT_TIMESTAMP_FMT = '%Y%m%dT%H%M%SZ'


def main(_):
    """Run the scanner."""
    audit_base_dir = os.path.dirname(__file__) + '/audit/'

    if FLAGS.list_engines:
        glob_string = audit_base_dir + '*engine*.py'
        engines = glob.glob(glob_string)
        print 'Available Forseti scanner engines:'
        for engine in engines:
            if 'base_rules_engine.py' not in engine:
                print os.path.basename(engine)
        sys.exit(1)

    if not FLAGS.use_engine:
        LOGGER.warn('Provide an engine file')
        sys.exit(1)
    else:
        rules_engine_filename = FLAGS.use_engine

    rules_engine_class = _create_rules_engine(audit_base_dir,
                                              rules_engine_filename)

    LOGGER.info('Using rules engine: %s', rules_engine_filename)

    LOGGER.info('Initializing the rules engine:\nUsing rules: %s', FLAGS.rules)

    if not FLAGS.rules:
        LOGGER.warn(('Provide a rules file. '
                     'Use "forseti_scanner --helpfull" for help.'))
        sys.exit(1)

    rules_engine = rules_engine_class(rules_file_path=FLAGS.rules)
    rules_engine.build_rule_book()

    snapshot_timestamp = _get_timestamp()
    if not snapshot_timestamp:
        LOGGER.warn('No snapshot timestamp found. Exiting.')
        sys.exit()

    # TODO: make this generic
    org_policies = _get_org_policies(snapshot_timestamp)
    project_policies = _get_project_policies(snapshot_timestamp)

    if not org_policies and not project_policies:
        LOGGER.warn('No policies found. Exiting.')
        sys.exit()

    all_violations = _find_violations(
        itertools.chain(
            org_policies.iteritems(),
            project_policies.iteritems()),
        rules_engine)

    # If there are violations, send results.
    if all_violations:
        resource_counts = {
            ResourceType.ORGANIZATION: len(org_policies),
            ResourceType.PROJECT: len(project_policies),
        }
        _output_results(all_violations,
                        snapshot_timestamp,
                        resource_counts=resource_counts)

    LOGGER.info('Done!')

def _import_rules_engine(path, name):
    """Load rules engine

    Args:
        path: System path of module.
        name: Name of the file.

    Returns:
        Module.
    """
    name, ext = os.path.splitext(name)
    try:
        file, filename, data = imp.find_module(name, [path])
        module = imp.load_module(name, file, filename, data)
    except ImportError as err:
        LOGGER.error('Failed to import module %s', name)
    return module

def _create_rules_engine(audit_base_dir, rules_engine_filename):
    """Create the rules engine class

    Args:
        audit_base_dir: Base directory with enignes.
        rules_engine_filename: Rules engine python file.

    Returns:
        Rules engined class.
    """
    rules_engine_class_name = None

    try:
        class_file = open(audit_base_dir + rules_engine_filename)
    except IOError as err:
        LOGGER.error('Error opening module file: %s', err)
        sys.exit(1)

    module_tree = ast.parse(class_file.read())
    for node in ast.walk(module_tree):
        if isinstance(node, ast.ClassDef) and 'Engine' in node.name:
            rules_engine_class_name = node.name
    
    if rules_engine_class_name is None:
        LOGGER.error('Engine module %s wasn\'t loaded', rules_engine_filename)
        sys.exit(1)

    rules_engine_module = _import_rules_engine(audit_base_dir,
                                              rules_engine_filename)
    rules_engine_class = getattr(rules_engine_module, rules_engine_class_name)

    return rules_engine_class

def _find_violations(policies, rules_engine):
    """Find violations in the policies.

    Args:
        policies: The list of policies to find violations in.
        rules_engine: The rules engine to run.

    Returns:
        A list of violations.
    """

    all_violations = []
    LOGGER.info('Finding policy violations...')
    for (resource, policy) in policies:
        LOGGER.debug('%s => %s', resource, policy)
        violations = rules_engine.find_policy_violations(
            resource, policy)
        LOGGER.debug(violations)
        all_violations.extend(violations)
    return all_violations

def _get_output_filename(now_utc):
    """Create the output filename.

    Args:
        now_utc: The datetime now in UTC.

    Returns:
        The output filename for the csv, formatted with the now_utc timestamp.
    """
{'fraction_non_alphanumeric': 0.09225751339597732, 'fraction_numerical': 0.0017084724702958764, 'mean_word_length': 2.8568433662773285, 'pattern_counts': {'":': 0, '<': 25, '<?xml version=': 0, '>': 27, 'https://': 2, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '659732', 'n_tokens_mistral': 3788, 'n_tokens_neox': 3709, 'n_words': 1038}