<commit_before># Copyright 2020 The StackStorm Authors.
# Copyright 2019 Extreme Networks, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import logging
import re
import sys
import traceback
import jsonschema

from collections.abc import Collection, Mapping
from st2common.util import schema
from st2common.constants import action as action_constants
from st2common.constants.secrets import MASKED_ATTRIBUTE_VALUE


LOG = logging.getLogger(__name__)


def _validate_runner(runner_schema, result):
    LOG.debug("Validating runner output: %s", runner_schema)

    # runner's output is always an object.
    runner_schema = {
        "type": "object",
        "properties": runner_schema,
        "additionalProperties": False,
    }

    schema.validate(result, runner_schema, cls=schema.get_validator("custom"))


def _validate_action(action_schema, result, output_key):
    LOG.debug("Validating action output: %s", action_schema)

    final_result = result[output_key]

    # TODO: support action output_schema for non-objects
    #       (see also st2common/models/api/action.py)
    action_schema = {
        "type": "object",
        "properties": action_schema,
        "additionalProperties": False,
    }

    schema.validate(final_result, action_schema, cls=schema.get_validator("custom"))


def _get_masked_value(spec, value):
    # malformed schema
    if not isinstance(spec, Mapping):
        return value

    if spec.get("secret", False):
        return MASKED_ATTRIBUTE_VALUE

    kind = spec.get("type")

    if kind in ("boolean", "integer", "null", "number", "string"):
        # already checked for spec["secret"] above; nothing else to check.
        return value

    elif kind == "object":
        if not isinstance(value, Mapping):
            # we can't process it unless it matches the expected type
            return value

        properties_schema = spec.get("properties", {})
        if properties_schema and isinstance(properties_schema, Mapping):
            # properties is not empty or malformed
            for key, property_spec in properties_schema.items():
                if key in value:
                    value[key] = _get_masked_value(property_spec, value[key])
            unhandled_keys = set(value.keys()) - set(properties_schema.keys())
        else:
            # properties is empty or malformed
            unhandled_keys = set(value.keys())

        pattern_properties_schema = spec.get("patternProperties")
        if pattern_properties_schema and isinstance(pattern_properties_schema, Mapping):
            # patternProperties is not malformed
            for key_pattern, pattern_property_spec in pattern_properties_schema:
                key_re = re.compile(key_pattern)
                for key in list(unhandled_keys):
                    if key_re.search(key):
                        value[key] = _get_masked_value(
                            pattern_property_spec, value[key]
                        )
                        unhandled_keys.remove(key)

        additional_properties_schema = spec.get("additionalProperties")
        if additional_properties_schema and isinstance(
            additional_properties_schema, Mapping
        ):
            # additionalProperties is a schema, not a boolean, and not malformed
            for key in unhandled_keys:
                value[key] = _get_masked_value(additional_properties_schema, value[key])

        return value

    elif kind == "array":
        if not isinstance(value, Collection):
            # we can't process it unless it matches the expected type
            return value

        items_schema = spec.get("items", {})
        output_count = len(value)
        if isinstance(items_schema, Mapping):
            # explicit schema for each item
            for i, item_spec in enumerate(items_schema):
                if i >= output_count:
                    break
                value[i] = _get_masked_value(item_spec, value[key])
            handled_count = len(items_schema)<commit_msg>check for unhandled_keys before compiling regex<commit_after># Copyright 2020 The StackStorm Authors.
# Copyright 2019 Extreme Networks, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import logging
import re
import sys
import traceback
import jsonschema

from collections.abc import Collection, Mapping
from st2common.util import schema
from st2common.constants import action as action_constants
from st2common.constants.secrets import MASKED_ATTRIBUTE_VALUE


LOG = logging.getLogger(__name__)


def _validate_runner(runner_schema, result):
    LOG.debug("Validating runner output: %s", runner_schema)

    # runner's output is always an object.
    runner_schema = {
        "type": "object",
        "properties": runner_schema,
        "additionalProperties": False,
    }

    schema.validate(result, runner_schema, cls=schema.get_validator("custom"))


def _validate_action(action_schema, result, output_key):
    LOG.debug("Validating action output: %s", action_schema)

    final_result = result[output_key]

    # TODO: support action output_schema for non-objects
    #       (see also st2common/models/api/action.py)
    action_schema = {
        "type": "object",
        "properties": action_schema,
        "additionalProperties": False,
    }

    schema.validate(final_result, action_schema, cls=schema.get_validator("custom"))


def _get_masked_value(spec, value):
    # malformed schema
    if not isinstance(spec, Mapping):
        return value

    if spec.get("secret", False):
        return MASKED_ATTRIBUTE_VALUE

    kind = spec.get("type")

    if kind in ("boolean", "integer", "null", "number", "string"):
        # already checked for spec["secret"] above; nothing else to check.
        return value

    elif kind == "object":
        if not isinstance(value, Mapping):
            # we can't process it unless it matches the expected type
            return value

        properties_schema = spec.get("properties", {})
        if properties_schema and isinstance(properties_schema, Mapping):
            # properties is not empty or malformed
            for key, property_spec in properties_schema.items():
                if key in value:
                    value[key] = _get_masked_value(property_spec, value[key])
            unhandled_keys = set(value.keys()) - set(properties_schema.keys())
        else:
            # properties is empty or malformed
            unhandled_keys = set(value.keys())

        pattern_properties_schema = spec.get("patternProperties")
        if (
            unhandled_keys
            and pattern_properties_schema
            and isinstance(pattern_properties_schema, Mapping)
        ):
            # patternProperties is not malformed
            for key_pattern, pattern_property_spec in pattern_properties_schema:
                if not unhandled_keys:
                    # nothing to check, don't compile the next pattern
                    break
                key_re = re.compile(key_pattern)
                for key in list(unhandled_keys):
                    if key_re.search(key):
                        value[key] = _get_masked_value(
                            pattern_property_spec, value[key]
                        )
                        unhandled_keys.remove(key)

        additional_properties_schema = spec.get("additionalProperties")
        if (
            unhandled_keys
            and additional_properties_schema
            and isinstance(additional_properties_schema, Mapping)
        ):
            # additionalProperties is a schema, not a boolean, and not malformed
            for key in unhandled_keys:
                value[key] = _get_masked_value(additional_properties_schema, value[key])

        return value

    elif kind == "array":
        if not isinstance(value, Collection):
            # we can't process it unless it matches the expected type
            return value

        items_schema = spec.get("items", {})
        output_count = len(value)
        if isinstance(items_schema, Mapping):
            # explicit schema for each item
            for i, item_spec in enumerate(items_schema):
                if i >= output_count:
                    break
                value[i] = _get_masked_value(item_spec, value[key])
            handled_count = len(items_schema)
{'fraction_non_alphanumeric': 0.07778015856010286, 'fraction_numerical': 0.0034283265481037067, 'mean_word_length': 2.5752585216392188, 'pattern_counts': {'":': 16, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5313205', 'n_tokens_mistral': 2452, 'n_tokens_neox': 2372, 'n_words': 753}