<commit_before>#     Copyright 2014 Netflix, Inc.
#
#     Licensed under the Apache License, Version 2.0 (the "License");
#     you may not use this file except in compliance with the License.
#     You may obtain a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#     Unless required by applicable law or agreed to in writing, software
#     distributed under the License is distributed on an "AS IS" BASIS,
#     WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#     See the License for the specific language governing permissions and
#     limitations under the License.
"""
.. module: security_monkey.auditor
    :platform: Unix
    :synopsis: This class is subclassed to add audit rules.

.. version:: $$VERSION$$
.. moduleauthor:: <NAME> <<EMAIL>>

"""
from six import string_types, text_type

from security_monkey import app, datastore, db
from security_monkey.watcher import ChangeItem, ensure_item_has_latest_revision_id
from security_monkey.common.jinja import get_jinja_env
from security_monkey.datastore import User, AuditorSettings, Item, ItemAudit, Technology, Account, ItemAuditScore, AccountPatternAuditScore
from security_monkey.common.utils import send_email
from security_monkey.account_manager import get_account_by_name
from security_monkey.alerters.custom_alerter import report_auditor_changes
from security_monkey.datastore import Account, Item, Technology, NetworkWhitelistEntry
from policyuniverse.arn import ARN
from sqlalchemy import and_
from collections import defaultdict
from threading import Lock
import json
import netaddr
import ipaddr
import re


auditor_registry = defaultdict(list)


class Categories:
    """ Define common issue categories to maintain consistency. """
    # Resource Policies:
    INTERNET_ACCESSIBLE = 'Internet Accessible'
    INTERNET_ACCESSIBLE_NOTES = '{entity} Actions: {actions}'
    INTERNET_ACCESSIBLE_NOTES_SG = '{entity} Access: [{access}]'

    FRIENDLY_CROSS_ACCOUNT = 'Friendly Cross Account'
    FRIENDLY_CROSS_ACCOUNT_NOTES = '{entity} Actions: {actions}'
    FRIENDLY_CROSS_ACCOUNT_NOTES_SG = '{entity} Access: [{access}]'

    THIRDPARTY_CROSS_ACCOUNT = 'Thirdparty Cross Account'
    THIRDPARTY_CROSS_ACCOUNT_NOTES = '{entity} Actions: {actions}'
    THIRDPARTY_CROSS_ACCOUNT_NOTES_SG = '{entity} Access: [{access}]'

    UNKNOWN_ACCESS = 'Unknown Access'
    UNKNOWN_ACCESS_NOTES = '{entity} Actions: {actions}'
    UNKNOWN_ACCESS_NOTES_SG = '{entity} Access: [{access}]'

    PARSE_ERROR = 'Parse Error'
    PARSE_ERROR_NOTES = 'Could not parse {input_type} - {input}'

    CROSS_ACCOUNT_ROOT = 'Cross-Account Root IAM'
    CROSS_ACCOUNT_ROOT_NOTES = '{entity} Actions: {actions}'

    # IAM Policies:
    ADMIN_ACCESS = 'Administrator Access'
    ADMIN_ACCESS_NOTES = 'Actions: {actions} Resources: {resource}'

    SENSITIVE_PERMISSIONS = 'Sensitive Permissions'
    SENSITIVE_PERMISSIONS_NOTES_1 = 'Actions: {actions} Resources: {resource}'
    SENSITIVE_PERMISSIONS_NOTES_2 = 'Service [{service}] Category: [{category}] Resources: {resource}'

    STATEMENT_CONSTRUCTION = 'Awkward Statement Construction'
    STATEMENT_CONSTRUCTION_NOTES = 'Construct: {construct}'

    # Anywhere
    INFORMATIONAL = 'Informational'
    INFORMATIONAL_NOTES = '{description}{specific}'

    ROTATION = 'Needs Rotation'
    ROTATION_NOTES = '{what} last rotated {requirement} on {date}'

    UNUSED = 'Unused Access'
    UNUSED_NOTES = '{what} last used {requirement} on {date}'

    INSECURE_CONFIGURATION = 'Insecure Configuration'
    INSECURE_CONFIGURATION_NOTES = '{description}'

    RECOMMENDATION = 'Recommendation'
    RECOMMENDATION_NOTES = '{description}'

    INSECURE_TLS = 'Insecure TLS'
    INSECURE_TLS_NOTES = 'Policy: [{policy}] Port: {port} Reason: [{reason}]'
    INSECURE_TLS_NOTES_2 = 'Policy: [{policy}] Port: {port} Reason: [{reason}] CVE: [{cve}]'

    # TODO
    # 	INSECURE_CERTIFICATE = 'Insecure Certificate'


class Entity:
    """ Entity instances provide a place to map policy elements like s3:my_bucket to the related account. """
    def __init__(self, category, value, account_name=None, account_identifier=None):
        self.category = category
        self.value = value
        self.account_name = account_name
        self.account_identifier = account_identifier

    @staticmethod
    def from_tuple(entity_tuple):
        return Entity(category=entity_tuple.category, value=entity_tuple.value)

    def __str__(self):
        strval = ''
        if self.account_name or self.account_identifier:
            strval = 'Account: [{identifier}/{account_name}] '.format(identifier=self.account_identifier, account_name=self.account_name)
        strval += 'Entity: [{category}:{value}]'.format(category=self.category, value=self.value)
        return strval

    def __repr__(self):
        return self.__str__()

class AuditorType(type):
    def __init__(cls, name, bases, attrs):
        super(AuditorType, cls).__init__(name, bases, attrs)
        if cls.__name__ != 'Auditor' and cls.index:
            # Only want to register auditors explicitly loaded by find_modules
            if not '.' in cls.__module__:
                found = False
                for auditor in auditor_registry[cls.index]:
                    if auditor.__module__ == cls.__module__ and auditor.__name__ == cls.__name__:
                        found = True
                        break
                if not found:
                    app.logger.debug("Registering auditor {} {}.{}".format(cls.index, cls.__module__, cls.__name__))
                    auditor_registry[cls.index].append(cls)


def add(to, key, value):
    if not key:
        return
    if key in to:
        to[key].add(value)
    else:
        to[key] = set([value])


class Auditor(object):
    """
    This class (and subclasses really) run a number of rules against the configurations
    and look for any violations.  These violations are saved with the object and a report
    is made available via the Web UI and through email.
    """
    index = None          # Should be overridden
    i_am_singular = None  # Should be overridden
    i_am_plural = None    # Should be overridden
    __metaclass__ = AuditorType
    support_auditor_indexes = []<commit_msg>allow custom plugin auditor<commit_after>#     Copyright 2014 Netflix, Inc.
#
#     Licensed under the Apache License, Version 2.0 (the "License");
#     you may not use this file except in compliance with the License.
#     You may obtain a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#     Unless required by applicable law or agreed to in writing, software
#     distributed under the License is distributed on an "AS IS" BASIS,
#     WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#     See the License for the specific language governing permissions and
#     limitations under the License.
"""
.. module: security_monkey.auditor
    :platform: Unix
    :synopsis: This class is subclassed to add audit rules.

.. version:: $$VERSION$$
.. moduleauthor:: <NAME> <<EMAIL>>

"""
from six import string_types, text_type

from security_monkey import app, datastore, db
from security_monkey.watcher import ChangeItem, ensure_item_has_latest_revision_id
from security_monkey.common.jinja import get_jinja_env
from security_monkey.datastore import User, AuditorSettings, Item, ItemAudit, Technology, Account, ItemAuditScore, AccountPatternAuditScore
from security_monkey.common.utils import send_email
from security_monkey.account_manager import get_account_by_name
from security_monkey.alerters.custom_alerter import report_auditor_changes
from security_monkey.datastore import Account, Item, Technology, NetworkWhitelistEntry
from policyuniverse.arn import ARN
from sqlalchemy import and_
from collections import defaultdict
from threading import Lock
import json
import netaddr
import ipaddr
import re
import pkg_resources


auditor_registry = defaultdict(list)


class Categories:
    """ Define common issue categories to maintain consistency. """
    # Resource Policies:
    INTERNET_ACCESSIBLE = 'Internet Accessible'
    INTERNET_ACCESSIBLE_NOTES = '{entity} Actions: {actions}'
    INTERNET_ACCESSIBLE_NOTES_SG = '{entity} Access: [{access}]'

    FRIENDLY_CROSS_ACCOUNT = 'Friendly Cross Account'
    FRIENDLY_CROSS_ACCOUNT_NOTES = '{entity} Actions: {actions}'
    FRIENDLY_CROSS_ACCOUNT_NOTES_SG = '{entity} Access: [{access}]'

    THIRDPARTY_CROSS_ACCOUNT = 'Thirdparty Cross Account'
    THIRDPARTY_CROSS_ACCOUNT_NOTES = '{entity} Actions: {actions}'
    THIRDPARTY_CROSS_ACCOUNT_NOTES_SG = '{entity} Access: [{access}]'

    UNKNOWN_ACCESS = 'Unknown Access'
    UNKNOWN_ACCESS_NOTES = '{entity} Actions: {actions}'
    UNKNOWN_ACCESS_NOTES_SG = '{entity} Access: [{access}]'

    PARSE_ERROR = 'Parse Error'
    PARSE_ERROR_NOTES = 'Could not parse {input_type} - {input}'

    CROSS_ACCOUNT_ROOT = 'Cross-Account Root IAM'
    CROSS_ACCOUNT_ROOT_NOTES = '{entity} Actions: {actions}'

    # IAM Policies:
    ADMIN_ACCESS = 'Administrator Access'
    ADMIN_ACCESS_NOTES = 'Actions: {actions} Resources: {resource}'

    SENSITIVE_PERMISSIONS = 'Sensitive Permissions'
    SENSITIVE_PERMISSIONS_NOTES_1 = 'Actions: {actions} Resources: {resource}'
    SENSITIVE_PERMISSIONS_NOTES_2 = 'Service [{service}] Category: [{category}] Resources: {resource}'

    STATEMENT_CONSTRUCTION = 'Awkward Statement Construction'
    STATEMENT_CONSTRUCTION_NOTES = 'Construct: {construct}'

    # Anywhere
    INFORMATIONAL = 'Informational'
    INFORMATIONAL_NOTES = '{description}{specific}'

    ROTATION = 'Needs Rotation'
    ROTATION_NOTES = '{what} last rotated {requirement} on {date}'

    UNUSED = 'Unused Access'
    UNUSED_NOTES = '{what} last used {requirement} on {date}'

    INSECURE_CONFIGURATION = 'Insecure Configuration'
    INSECURE_CONFIGURATION_NOTES = '{description}'

    RECOMMENDATION = 'Recommendation'
    RECOMMENDATION_NOTES = '{description}'

    INSECURE_TLS = 'Insecure TLS'
    INSECURE_TLS_NOTES = 'Policy: [{policy}] Port: {port} Reason: [{reason}]'
    INSECURE_TLS_NOTES_2 = 'Policy: [{policy}] Port: {port} Reason: [{reason}] CVE: [{cve}]'

    # TODO
    # 	INSECURE_CERTIFICATE = 'Insecure Certificate'


class Entity:
    """ Entity instances provide a place to map policy elements like s3:my_bucket to the related account. """
    def __init__(self, category, value, account_name=None, account_identifier=None):
        self.category = category
        self.value = value
        self.account_name = account_name
        self.account_identifier = account_identifier

    @staticmethod
    def from_tuple(entity_tuple):
        return Entity(category=entity_tuple.category, value=entity_tuple.value)

    def __str__(self):
        strval = ''
        if self.account_name or self.account_identifier:
            strval = 'Account: [{identifier}/{account_name}] '.format(identifier=self.account_identifier, account_name=self.account_name)
        strval += 'Entity: [{category}:{value}]'.format(category=self.category, value=self.value)
        return strval

    def __repr__(self):
        return self.__str__()

class AuditorType(type):
    def __init__(cls, name, bases, attrs):
        super(AuditorType, cls).__init__(name, bases, attrs)
        if cls.__name__ != 'Auditor' and cls.index:
            # Only want to register auditors explicitly loaded by find_modules or entry points
            plugin_names = map(lambda x: x.name,pkg_resources.iter_entry_points('security_monkey.plugins'))
            if not '.' in cls.__module__ or cls.__module__ in plugin_names: 
                found = False
                for auditor in auditor_registry[cls.index]:
                    if auditor.__module__ == cls.__module__ and auditor.__name__ == cls.__name__:
                        found = True
                        break
                if not found:
                    app.logger.debug("Registering auditor {} {}.{}".format(cls.index, cls.__module__, cls.__name__))
                    auditor_registry[cls.index].append(cls)


def add(to, key, value):
    if not key:
        return
    if key in to:
        to[key].add(value)
    else:
        to[key] = set([value])


class Auditor(object):
    """
    This class (and subclasses really) run a number of rules against the configurations
    and look for any violations.  These violations are saved with the object and a report
    is made available via the Web UI and through email.
    """
    index = None          # Should be overridden
    i_am_singular = None  # Should be overridden
    i_am_plural = None    # Should be overridden
    __metaclass__ = AuditorType
    support_auditor_indexes = []
{'fraction_non_alphanumeric': 0.09090196078431373, 'fraction_numerical': 0.0018823529411764706, 'mean_word_length': 3.658750456704421, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6170628', 'n_tokens_mistral': 3866, 'n_tokens_neox': 3657, 'n_words': 1068}