<commit_before>#! /usr/bin/env python3
# SPDX-License-Identifier: Apache-2.0
#
# Copyright (C) 2018, ARM Limited and contributors.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

import argparse
import collections
import contextlib
import copy
import datetime
import hashlib
import inspect
import io
import itertools
import os
import pathlib
import random
import shutil
import sys

from exekall.customization import AdaptorBase
import exekall.utils as utils
from exekall.utils import NoValue, error, warn, debug, info, out, add_argument
import exekall.engine as engine

# Create an operator for all callables that have been detected in a given
# set of modules
def build_op_set(callable_pool, non_reusable_type_set, allowed_pattern_set, adaptor):
    op_set = {
        engine.Operator(
            callable_,
            non_reusable_type_set=non_reusable_type_set,
            tags_getter=adaptor.get_tags
        )
        for callable_ in callable_pool
    }

    filtered_op_set = adaptor.filter_op_set(op_set)
    # Make sure we have all the explicitely allowed operators
    filtered_op_set.update(
        op for op in op_set
        if utils.match_name(op.get_name(full_qual=True), allowed_pattern_set)
    )
    return filtered_op_set

def build_patch_map(sweep_spec_list, param_spec_list, op_set):
    sweep_spec_list = copy.copy(sweep_spec_list)
    # Make a sweep spec from a simpler param spec
    sweep_spec_list.extend(
        (callable_pattern, param, value, value, 1)
        for callable_pattern, param, value in param_spec_list
    )

    patch_map = dict()
    for callable_pattern, param, start, stop, step in sweep_spec_list:
        for op in op_set:
            callable_ = op.callable_
            callable_name = utils.get_name(callable_, full_qual=True)
            if not utils.match_name(callable_name, [callable_pattern]):
                continue
            patch_map.setdefault(op, dict())[param] = [
                i for i in utils.sweep_param(
                    callable_, param,
                    start, stop, step
                )
            ]
    return patch_map

def apply_patch_map(patch_map, adaptor):
    prebuilt_op_set = set()
    for op, param_patch_map in patch_map.items():
        try:
            new_op_set = op.force_param(
                param_patch_map,
                tags_getter=adaptor.get_tags
            )
            prebuilt_op_set.update(new_op_set)
        except KeyError as e:
            error('Callable "{callable_}" has no parameters {param}'.format(
                callable_=op.name,
                param=e.args[0]
            ))
            continue

    return prebuilt_op_set<commit_msg>exekall: Allow repeating --param for same parameter<commit_after>#! /usr/bin/env python3
# SPDX-License-Identifier: Apache-2.0
#
# Copyright (C) 2018, ARM Limited and contributors.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

import argparse
import collections
import contextlib
import copy
import datetime
import hashlib
import inspect
import io
import itertools
import os
import pathlib
import random
import shutil
import sys

from exekall.customization import AdaptorBase
import exekall.utils as utils
from exekall.utils import NoValue, error, warn, debug, info, out, add_argument
import exekall.engine as engine

# Create an operator for all callables that have been detected in a given
# set of modules
def build_op_set(callable_pool, non_reusable_type_set, allowed_pattern_set, adaptor):
    op_set = {
        engine.Operator(
            callable_,
            non_reusable_type_set=non_reusable_type_set,
            tags_getter=adaptor.get_tags
        )
        for callable_ in callable_pool
    }

    filtered_op_set = adaptor.filter_op_set(op_set)
    # Make sure we have all the explicitely allowed operators
    filtered_op_set.update(
        op for op in op_set
        if utils.match_name(op.get_name(full_qual=True), allowed_pattern_set)
    )
    return filtered_op_set

def build_patch_map(sweep_spec_list, param_spec_list, op_set):
    sweep_spec_list = copy.copy(sweep_spec_list)
    # Make a sweep spec from a simpler param spec
    sweep_spec_list.extend(
        (callable_pattern, param, value, value, 1)
        for callable_pattern, param, value in param_spec_list
    )

    patch_map = dict()
    for callable_pattern, param, start, stop, step in sweep_spec_list:
        for op in op_set:
            callable_ = op.callable_
            callable_name = utils.get_name(callable_, full_qual=True)
            if not utils.match_name(callable_name, [callable_pattern]):
                continue
            patch_map.setdefault(op, dict()).setdefault(param, []).extend(
                utils.sweep_param(
                    callable_, param,
                    start, stop, step
            ))
    return patch_map

def apply_patch_map(patch_map, adaptor):
    prebuilt_op_set = set()
    for op, param_patch_map in patch_map.items():
        try:
            new_op_set = op.force_param(
                param_patch_map,
                tags_getter=adaptor.get_tags
            )
            prebuilt_op_set.update(new_op_set)
        except KeyError as e:
            error('Callable "{callable_}" has no parameters {param}'.format(
                callable_=op.name,
                param=e.args[0]
            ))
            continue

    return prebuilt_op_set
{'fraction_non_alphanumeric': 0.0667294708745486, 'fraction_numerical': 0.004082273512325325, 'mean_word_length': 2.789411064842356, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2949772', 'n_tokens_mistral': 1898, 'n_tokens_neox': 1856, 'n_words': 578}