<commit_before># -*- coding: utf-8 -*-
# Copyright 2014 Objectif Libre
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.
#
# @author: <NAME>
#
import decimal

from cloudkitty.openstack.common import log as logging
from cloudkitty import rating
from cloudkitty.rating.hash.controllers import root as root_api
from cloudkitty.rating.hash.db import api as hash_db_api

LOG = logging.getLogger(__name__)


class HashMap(rating.RatingProcessorBase):
    """HashMap rating module.

    HashMap can be used to map arbitrary fields of a resource to different
    costs.
    """

    module_name = 'hashmap'
    description = 'HashMap rating module.'
    hot_config = True
    config_controller = root_api.HashMapConfigController

    db_api = hash_db_api.get_instance()

    def __init__(self, tenant_id=None):
        super(HashMap, self).__init__(tenant_id)
        self._entries = {}
        self._res = {}
        self._load_rates()

    def reload_config(self):
        """Reload the module's configuration.

        """
        self._load_rates()

    def _load_mappings(self, mappings_uuid_list):
        hashmap = hash_db_api.get_instance()
        mappings = {}
        for mapping_uuid in mappings_uuid_list:
            mapping_db = hashmap.get_mapping(uuid=mapping_uuid)
            if mapping_db.group_id:
                group_name = mapping_db.group.name
            else:
                group_name = '_DEFAULT_'
            if group_name not in mappings:
                mappings[group_name] = {}
            current_scope = mappings[group_name]

            mapping_value = mapping_db.value
            if mapping_value:
                current_scope[mapping_value] = {}
                current_scope = current_scope[mapping_value]
            current_scope['type'] = mapping_db.map_type
            current_scope['cost'] = mapping_db.cost
        return mappings

    def _load_thresholds(self, thresholds_uuid_list):
        hashmap = hash_db_api.get_instance()
        thresholds = {}
        for threshold_uuid in thresholds_uuid_list:
            threshold_db = hashmap.get_threshold(uuid=threshold_uuid)
            if threshold_db.group_id:
                group_name = threshold_db.group.name
            else:
                group_name = '_DEFAULT_'
            if group_name not in thresholds:
                thresholds[group_name] = {}
            current_scope = thresholds[group_name]

            threshold_level = threshold_db.level
            current_scope[threshold_level] = {}
            current_scope = current_scope[threshold_level]
            current_scope['type'] = threshold_db.map_type
            current_scope['cost'] = threshold_db.cost
        return thresholds

    def _load_service_entries(self, service_name, service_uuid):
        hashmap = hash_db_api.get_instance()
        self._entries[service_name] = {}
        mappings_uuid_list = hashmap.list_mappings(
            service_uuid=service_uuid)
        mappings = self._load_mappings(mappings_uuid_list)
        self._entries[service_name]['mappings'] = mappings
        thresholds_uuid_list = hashmap.list_thresholds(
            service_uuid=service_uuid)
        thresholds = self._load_thresholds(thresholds_uuid_list)
        self._entries[service_name]['thresholds'] = thresholds

    def _load_field_entries(self, service_name, field_name, field_uuid):
        hashmap = hash_db_api.get_instance()
        mappings_uuid_list = hashmap.list_mappings(field_uuid=field_uuid)
        mappings = self._load_mappings(mappings_uuid_list)
        thresholds_uuid_list = hashmap.list_thresholds(field_uuid=field_uuid)
        thresholds = self._load_thresholds(thresholds_uuid_list)
        if service_name not in self._entries:
            self._entries[service_name] = {}
        if 'fields' not in self._entries[service_name]:
            self._entries[service_name]['fields'] = {}
        scope = self._entries[service_name]['fields'][field_name] = {}
        scope['mappings'] = mappings
        scope['thresholds'] = thresholds

    def _load_rates(self):
        self._entries = {}
        hashmap = hash_db_api.get_instance()
        services_uuid_list = hashmap.list_services()
        for service_uuid in services_uuid_list:
            service_db = hashmap.get_service(uuid=service_uuid)
            service_name = service_db.name
            self._load_service_entries(service_name, service_uuid)
            fields_uuid_list = hashmap.list_fields(service_uuid)
            for field_uuid in fields_uuid_list:
                field_db = hashmap.get_field(uuid=field_uuid)
                field_name = field_db.name
                self._load_field_entries(service_name, field_name, field_uuid)

    def add_rating_informations(self, data):
        if 'rating' not in data:
            data['rating'] = {'price': 0}
        for entry in self._res.values():
            rate = entry['rate']
            flat = entry['flat']
            if entry['threshold']['scope'] == 'field':
                if entry['threshold']['type'] == 'flat':
                    flat += entry['threshold']['cost']
                else:
                    rate *= entry['threshold']['cost']
            res = rate * flat
            res *= data['vol']['qty']
            if entry['threshold']['scope'] == 'service':
                if entry['threshold']['type'] == 'flat':
                    res += entry['threshold']['cost']
                else:
                    res *= entry['threshold']['cost']
            data['rating']['price'] += res

    def update_result(self,
                      group,
                      map_type,
                      cost,
                      level=0,
                      is_threshold=False,
                      threshold_scope='field'):
        if group not in self._res:
            self._res[group] = {'flat': 0,
                                'rate': 1,
                                'threshold': {
                                    'level': -1,
                                    'cost': 0,
                                    'type': 'flat',
                                    'scope': 'field'}}
        if is_threshold:
            best = self._res[group]['threshold']['level']
            if level > best:
                self._res[group]['threshold']['level'] = level
                self._res[group]['threshold']['cost'] = cost
                self._res[group]['threshold']['type'] = map_type
                self._res[group]['threshold']['scope'] = threshold_scope
        else:
            if map_type == 'rate':
                self._res[group]['rate'] *= cost
            elif map_type == 'flat':
                new_flat = cost
                cur_flat = self._res[group]['flat']
                if new_flat > cur_flat:
                    self._res[group]['flat'] = new_flat

    def process_mappings(self,
                         mapping_groups,
                         cmp_value):
        for group_name, mappings in mapping_groups.items():
            mapping_default = mappings.get('_DEFAULT_', {})
            matched = False
            for mapping_value, mapping in mappings.items():
                if mapping is mapping_default:
                    continue
                if cmp_value == mapping_value:
                    self.update_result(
                        group_name,
                        mapping['type'],
                        mapping['cost'])
                    matched = True
            if not matched and mapping_default:
                self.update_result(
                    group_name,
                    mapping_default['type'],
                    mapping_default['cost'])

    def process_thresholds(self,
                           threshold_groups,
                           cmp_level,
                           threshold_type):
        for group_name, thresholds in threshold_groups.items():
            threshold_default = thresholds.get('_DEFAULT_', {})
            matched = False
            for threshold_level, threshold in thresholds.items():
                if threshold is threshold_default:
                    continue
                if cmp_level >= threshold_level:
                    self.update_result(
                        group_name,
                        threshold['type'],
                        threshold['cost'],
                        threshold_level,
                        True,
                        threshold_type)
                    matched = True
            if not matched and threshold_default:
                self.update_result(
                    group_name,
                    threshold_default['type'],
                    threshold_default['cost'],
                    True,
                    threshold_type)

    def process_services(self, service_name, data):
        if service_name not in self._entries:
            return
        service_mappings = self._entries[service_name]['mappings']
        for group_name, mapping in service_mappings.items():
            self.update_result(group_name,
                               mapping['type'],
                               mapping['cost'])
        service_thresholds = self._entries[service_name]['thresholds']
        self.process_thresholds(service_thresholds,
                                data['vol']['qty'],
                                'service')

    def process_fields(self, service_name, data):
        if service_name not in self._entries:
            return
        desc_data = data['desc']
        field_mappings = self._entries[service_name]['fields']
        for field_name, group_mappings in field_mappings.items():
            if field_name not in desc_data:
                continue
            cmp_value = desc_data[field_name]
            self.process_mappings(group_mappings['mappings'],
                                  cmp_value)
            if group_mappings['thresholds']:
                self.process_thresholds(group_mappings['thresholds'],
                                        decimal.Decimal(cmp_value),
                                        'field')

    def process(self, data):
        for cur_data in data:
            cur_usage = cur_data['usage']
            for service_name, service_data in cur_usage.items():
                for item in service_data:
                    self._res = {}
                    self.process_services(service_name, item)
                    self.process_fields(service_name, item)
                    self.add_rating_informations(item)
        return data
<commit_msg>Added implicit cast to decimal in rating total calculation<commit_after># -*- coding: utf-8 -*-
# Copyright 2014 <NAME>
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.
#
# @author: <NAME>
#
import decimal

from cloudkitty.openstack.common import log as logging
from cloudkitty import rating
from cloudkitty.rating.hash.controllers import root as root_api
from cloudkitty.rating.hash.db import api as hash_db_api

LOG = logging.getLogger(__name__)


class HashMap(rating.RatingProcessorBase):
    """HashMap rating module.

    HashMap can be used to map arbitrary fields of a resource to different
    costs.
    """

    module_name = 'hashmap'
    description = 'HashMap rating module.'
    hot_config = True
    config_controller = root_api.HashMapConfigController

    db_api = hash_db_api.get_instance()

    def __init__(self, tenant_id=None):
        super(HashMap, self).__init__(tenant_id)
        self._entries = {}
        self._res = {}
        self._load_rates()

    def reload_config(self):
        """Reload the module's configuration.

        """
        self._load_rates()

    def _load_mappings(self, mappings_uuid_list):
        hashmap = hash_db_api.get_instance()
        mappings = {}
        for mapping_uuid in mappings_uuid_list:
            mapping_db = hashmap.get_mapping(uuid=mapping_uuid)
            if mapping_db.group_id:
                group_name = mapping_db.group.name
            else:
                group_name = '_DEFAULT_'
            if group_name not in mappings:
                mappings[group_name] = {}
            current_scope = mappings[group_name]

            mapping_value = mapping_db.value
            if mapping_value:
                current_scope[mapping_value] = {}
                current_scope = current_scope[mapping_value]
            current_scope['type'] = mapping_db.map_type
            current_scope['cost'] = mapping_db.cost
        return mappings

    def _load_thresholds(self, thresholds_uuid_list):
        hashmap = hash_db_api.get_instance()
        thresholds = {}
        for threshold_uuid in thresholds_uuid_list:
            threshold_db = hashmap.get_threshold(uuid=threshold_uuid)
            if threshold_db.group_id:
                group_name = threshold_db.group.name
            else:
                group_name = '_DEFAULT_'
            if group_name not in thresholds:
                thresholds[group_name] = {}
            current_scope = thresholds[group_name]

            threshold_level = threshold_db.level
            current_scope[threshold_level] = {}
            current_scope = current_scope[threshold_level]
            current_scope['type'] = threshold_db.map_type
            current_scope['cost'] = threshold_db.cost
        return thresholds

    def _load_service_entries(self, service_name, service_uuid):
        hashmap = hash_db_api.get_instance()
        self._entries[service_name] = {}
        mappings_uuid_list = hashmap.list_mappings(
            service_uuid=service_uuid)
        mappings = self._load_mappings(mappings_uuid_list)
        self._entries[service_name]['mappings'] = mappings
        thresholds_uuid_list = hashmap.list_thresholds(
            service_uuid=service_uuid)
        thresholds = self._load_thresholds(thresholds_uuid_list)
        self._entries[service_name]['thresholds'] = thresholds

    def _load_field_entries(self, service_name, field_name, field_uuid):
        hashmap = hash_db_api.get_instance()
        mappings_uuid_list = hashmap.list_mappings(field_uuid=field_uuid)
        mappings = self._load_mappings(mappings_uuid_list)
        thresholds_uuid_list = hashmap.list_thresholds(field_uuid=field_uuid)
        thresholds = self._load_thresholds(thresholds_uuid_list)
        if service_name not in self._entries:
            self._entries[service_name] = {}
        if 'fields' not in self._entries[service_name]:
            self._entries[service_name]['fields'] = {}
        scope = self._entries[service_name]['fields'][field_name] = {}
        scope['mappings'] = mappings
        scope['thresholds'] = thresholds

    def _load_rates(self):
        self._entries = {}
        hashmap = hash_db_api.get_instance()
        services_uuid_list = hashmap.list_services()
        for service_uuid in services_uuid_list:
            service_db = hashmap.get_service(uuid=service_uuid)
            service_name = service_db.name
            self._load_service_entries(service_name, service_uuid)
            fields_uuid_list = hashmap.list_fields(service_uuid)
            for field_uuid in fields_uuid_list:
                field_db = hashmap.get_field(uuid=field_uuid)
                field_name = field_db.name
                self._load_field_entries(service_name, field_name, field_uuid)

    def add_rating_informations(self, data):
        if 'rating' not in data:
            data['rating'] = {'price': 0}
        for entry in self._res.values():
            rate = entry['rate']
            flat = entry['flat']
            if entry['threshold']['scope'] == 'field':
                if entry['threshold']['type'] == 'flat':
                    flat += entry['threshold']['cost']
                else:
                    rate *= entry['threshold']['cost']
            res = rate * flat
            # FIXME(sheeprine): Added here to ensure that qty is decimal
            res *= decimal.Decimal(data['vol']['qty'])
            if entry['threshold']['scope'] == 'service':
                if entry['threshold']['type'] == 'flat':
                    res += entry['threshold']['cost']
                else:
                    res *= entry['threshold']['cost']
            data['rating']['price'] += res

    def update_result(self,
                      group,
                      map_type,
                      cost,
                      level=0,
                      is_threshold=False,
                      threshold_scope='field'):
        if group not in self._res:
            self._res[group] = {'flat': 0,
                                'rate': 1,
                                'threshold': {
                                    'level': -1,
                                    'cost': 0,
                                    'type': 'flat',
                                    'scope': 'field'}}
        if is_threshold:
            best = self._res[group]['threshold']['level']
            if level > best:
                self._res[group]['threshold']['level'] = level
                self._res[group]['threshold']['cost'] = cost
                self._res[group]['threshold']['type'] = map_type
                self._res[group]['threshold']['scope'] = threshold_scope
        else:
            if map_type == 'rate':
                self._res[group]['rate'] *= cost
            elif map_type == 'flat':
                new_flat = cost
                cur_flat = self._res[group]['flat']
                if new_flat > cur_flat:
                    self._res[group]['flat'] = new_flat

    def process_mappings(self,
                         mapping_groups,
                         cmp_value):
        for group_name, mappings in mapping_groups.items():
            mapping_default = mappings.get('_DEFAULT_', {})
            matched = False
            for mapping_value, mapping in mappings.items():
                if mapping is mapping_default:
                    continue
                if cmp_value == mapping_value:
                    self.update_result(
                        group_name,
                        mapping['type'],
                        mapping['cost'])
                    matched = True
            if not matched and mapping_default:
                self.update_result(
                    group_name,
                    mapping_default['type'],
                    mapping_default['cost'])

    def process_thresholds(self,
                           threshold_groups,
                           cmp_level,
                           threshold_type):
        for group_name, thresholds in threshold_groups.items():
            threshold_default = thresholds.get('_DEFAULT_', {})
            matched = False
            for threshold_level, threshold in thresholds.items():
                if threshold is threshold_default:
                    continue
                if cmp_level >= threshold_level:
                    self.update_result(
                        group_name,
                        threshold['type'],
                        threshold['cost'],
                        threshold_level,
                        True,
                        threshold_type)
                    matched = True
            if not matched and threshold_default:
                self.update_result(
                    group_name,
                    threshold_default['type'],
                    threshold_default['cost'],
                    True,
                    threshold_type)

    def process_services(self, service_name, data):
        if service_name not in self._entries:
            return
        service_mappings = self._entries[service_name]['mappings']
        for group_name, mapping in service_mappings.items():
            self.update_result(group_name,
                               mapping['type'],
                               mapping['cost'])
        service_thresholds = self._entries[service_name]['thresholds']
        self.process_thresholds(service_thresholds,
                                data['vol']['qty'],
                                'service')

    def process_fields(self, service_name, data):
        if service_name not in self._entries:
            return
        desc_data = data['desc']
        field_mappings = self._entries[service_name]['fields']
        for field_name, group_mappings in field_mappings.items():
            if field_name not in desc_data:
                continue
            cmp_value = desc_data[field_name]
            self.process_mappings(group_mappings['mappings'],
                                  cmp_value)
            if group_mappings['thresholds']:
                self.process_thresholds(group_mappings['thresholds'],
                                        decimal.Decimal(cmp_value),
                                        'field')

    def process(self, data):
        for cur_data in data:
            cur_usage = cur_data['usage']
            for service_name, service_data in cur_usage.items():
                for item in service_data:
                    self._res = {}
                    self.process_services(service_name, item)
                    self.process_fields(service_name, item)
                    self.add_rating_informations(item)
        return data

{'fraction_non_alphanumeric': 0.08844514247251514, 'fraction_numerical': 0.0013461969934933811, 'mean_word_length': 1.7428923076923077, 'pattern_counts': {'":': 0, '<': 6, '<?xml version=': 0, '>': 12, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 14, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6699565', 'n_tokens_mistral': 5782, 'n_tokens_neox': 5460, 'n_words': 1220}