<commit_before># -*- coding: utf-8 -*-

#    Copyright 2013 Mirantis, Inc.
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

'''
Classes for working with disks and volumes.
All sizes in megabytes.
'''

import json

from copy import deepcopy
from functools import partial
from nailgun.errors import errors
from nailgun.logger import logger


def only_disks(spaces):
    '''Helper for retrieving only disks from spaces
    '''
    return filter(lambda space: space['type'] == 'disk', spaces)


def only_vg(spaces):
    '''Helper for retrieving only volumes groups from spaces
    '''
    return filter(lambda space: space['type'] == 'vg', spaces)


def gb_to_mb(gb):
    '''Convert gigabytes to megabytes
    '''
    return int(gb * 1024)


def byte_to_megabyte(byte):
    '''Convert bytes to megabytes
    '''
    return byte / 1024 ** 2


def get_node_volumes(node):
    """Helper for retrieving node volumes in correct order.
    If spaces don't defained for role, will be used
    partitioning for role `other`.
    """
    node_volumes = []

    role_volumes = node.cluster.release.volumes_metadata[
        'volumes_roles_mapping']
    roles = node.roles + node.pending_roles

    for role in roles:
        if not role_volumes.get(role):
            continue

        for volume in role_volumes[role]:
            if volume not in node_volumes:
                node_volumes.append(volume)

    # Use role other
    if not node_volumes:
        logger.warn('Cannot find volumes for node: %s assigning default '
                    'volumes' % (node.full_name))
        for volume in role_volumes['other']:
            node_volumes.append(volume)

    return node_volumes


class DisksFormatConvertor(object):
    '''Class converts format from `simple` in which we
    communicate with UI to `full` in which we store
    data about disks\volumes in database, send to
    orchestrator and vice versa.

    Full disk format example:
        [
            {
                "type": "disk",
                "id": "sda",
                "size": 953869,
                "volumes": [
                    {
                        "mount": "/boot",
                        "type": "raid",
                        "size": 200
                    },
                    .....
                    {
                        "size": 938905,
                        "type": "pv",
                        "vg": "os"
                    }
                ]
            }
        ]

    Simple disk format example:
        [
            {
                "id": "sda",
                "size": 953869,
                "volumes": [
                    {
                        "name": "os",
                        "size": 938905,
                    }
                ]
            }
        ]
    '''

    @classmethod
    def format_disks_to_full(cls, node, disks):
        '''convert disks from simple format to full format
        '''
        full_format = []
        volume_manager = node.volume_manager
        for disk in disks:
            for volume in disk['volumes']:
                full_format = volume_manager.set_volume_size(
                    disk['id'], volume['name'], volume['size'])

        return full_format

    @classmethod
    def format_disks_to_simple(cls, full):
        '''convert disks from full format to simple format
        '''
        disks_in_simple_format = []

        # retrieve only physical disks
        disks_full_format = only_disks(full)

        for disk in disks_full_format:
            reserved_size = cls.calculate_service_partitions_size(
                disk['volumes'])

            lvm_pvs_size = sum([
                volume.get('lvm_meta_size', 0) for volume in disk['volumes']])
            size = 0
            if disk['size'] >= reserved_size:
                size = disk['size'] - reserved_size - lvm_pvs_size

            disk_simple = {
                'id': disk['id'],
                'name': disk['name'],
                'size': size,
                'volumes': cls.serialize_volumes(disk['volumes'])}

            disks_in_simple_format.append(disk_simple)

        return disks_in_simple_format

    @classmethod
    def calculate_service_partitions_size(self, volumes):
        service_partitions = filter(
            lambda vg: vg.get('type') != 'pv' and
            vg.get('type') != 'partition' and
            (vg.get('type') != 'raid' or vg.get('mount') == '/boot'),
            volumes)

        return sum(
            [partition.get('size', 0) for partition in service_partitions])

    @classmethod
    def serialize_volumes(cls, all_partitions):
        """Convert volumes from full format to simple format
        """
        pv_full_format = filter(
            lambda vg: vg.get('type') == 'pv', all_partitions)

        partitions_full_format = filter(
            lambda vg: vg.get('type') == 'partition', all_partitions)

        raid_full_format = filter(
            lambda vg: vg.get('type') == 'raid' and vg.get('mount') != '/boot',
            all_partitions)

        volumes_simple_format = []
        for volume in pv_full_format:
            calculated_size = volume['size'] - volume['lvm_meta_size']
            size = calculated_size if calculated_size > 0 else 0

            volume_simple = {
                'name': volume['vg'],
                'size': size}

            volumes_simple_format.append(volume_simple)

        for partition in partitions_full_format:
            volumes_simple_format.append({
                'name': partition['name'],<commit_msg>consistently exclude boot volume in disks API<commit_after># -*- coding: utf-8 -*-

#    Copyright 2013 Mirantis, Inc.
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

'''
Classes for working with disks and volumes.
All sizes in megabytes.
'''

import json

from copy import deepcopy
from functools import partial
from nailgun.errors import errors
from nailgun.logger import logger


def is_service(space):
    '''Helper to check if the space is a service partition
    '''
    return (space.get('mount') == '/boot' or
            space.get('type') not in ('pv', 'partition', 'raid'))


def only_disks(spaces):
    '''Helper for retrieving only disks from spaces
    '''
    return filter(lambda space: space['type'] == 'disk', spaces)


def only_vg(spaces):
    '''Helper for retrieving only volumes groups from spaces
    '''
    return filter(lambda space: space['type'] == 'vg', spaces)


def gb_to_mb(gb):
    '''Convert gigabytes to megabytes
    '''
    return int(gb * 1024)


def byte_to_megabyte(byte):
    '''Convert bytes to megabytes
    '''
    return byte / 1024 ** 2


def get_node_volumes(node):
    """Helper for retrieving node volumes in correct order.
    If spaces don't defained for role, will be used
    partitioning for role `other`.
    """
    node_volumes = []

    role_volumes = node.cluster.release.volumes_metadata[
        'volumes_roles_mapping']
    roles = node.roles + node.pending_roles

    for role in roles:
        if not role_volumes.get(role):
            continue

        for volume in role_volumes[role]:
            if volume not in node_volumes:
                node_volumes.append(volume)

    # Use role other
    if not node_volumes:
        logger.warn('Cannot find volumes for node: %s assigning default '
                    'volumes' % (node.full_name))
        for volume in role_volumes['other']:
            node_volumes.append(volume)

    return node_volumes


class DisksFormatConvertor(object):
    '''Class converts format from `simple` in which we
    communicate with UI to `full` in which we store
    data about disks\volumes in database, send to
    orchestrator and vice versa.

    Full disk format example:
        [
            {
                "type": "disk",
                "id": "sda",
                "size": 953869,
                "volumes": [
                    {
                        "mount": "/boot",
                        "type": "raid",
                        "size": 200
                    },
                    .....
                    {
                        "size": 938905,
                        "type": "pv",
                        "vg": "os"
                    }
                ]
            }
        ]

    Simple disk format example:
        [
            {
                "id": "sda",
                "size": 953869,
                "volumes": [
                    {
                        "name": "os",
                        "size": 938905,
                    }
                ]
            }
        ]
    '''

    @classmethod
    def format_disks_to_full(cls, node, disks):
        '''convert disks from simple format to full format
        '''
        full_format = []
        volume_manager = node.volume_manager
        for disk in disks:
            for volume in disk['volumes']:
                full_format = volume_manager.set_volume_size(
                    disk['id'], volume['name'], volume['size'])

        return full_format

    @classmethod
    def format_disks_to_simple(cls, full):
        '''convert disks from full format to simple format
        '''
        disks_in_simple_format = []

        # retrieve only physical disks
        disks_full_format = only_disks(full)

        for disk in disks_full_format:
            reserved_size = cls.calculate_service_partitions_size(
                disk['volumes'])

            lvm_pvs_size = sum([
                volume.get('lvm_meta_size', 0) for volume in disk['volumes']])
            size = 0
            if disk['size'] >= reserved_size:
                size = disk['size'] - reserved_size - lvm_pvs_size

            disk_simple = {
                'id': disk['id'],
                'name': disk['name'],
                'size': size,
                'volumes': cls.serialize_volumes(disk['volumes'])}

            disks_in_simple_format.append(disk_simple)

        return disks_in_simple_format

    @classmethod
    def calculate_service_partitions_size(self, volumes):
        service_partitions = filter(is_service, volumes)
        return sum(
            [partition.get('size', 0) for partition in service_partitions])

    @classmethod
    def serialize_volumes(cls, all_partitions):
        """Convert volumes from full format to simple format
        """
        non_service_volumes = filter(
            lambda vg: not is_service(vg), all_partitions)

        pv_full_format = filter(
            lambda vg: vg.get('type') == 'pv', non_service_volumes)

        partitions_full_format = filter(
            lambda vg: vg.get('type') == 'partition', non_service_volumes)

        raid_full_format = filter(
            lambda vg: vg.get('type') == 'raid', non_service_volumes)

        volumes_simple_format = []
        for volume in pv_full_format:
            calculated_size = volume['size'] - volume['lvm_meta_size']
            size = calculated_size if calculated_size > 0 else 0

            volume_simple = {
                'name': volume['vg'],
                'size': size}

            volumes_simple_format.append(volume_simple)

        for partition in partitions_full_format:
            volumes_simple_format.append({
                'name': partition['name'],
{'fraction_non_alphanumeric': 0.09836465706614596, 'fraction_numerical': 0.008136034496786267, 'mean_word_length': 1.837488457987073, 'pattern_counts': {'":': 30, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5609703', 'n_tokens_mistral': 3522, 'n_tokens_neox': 3270, 'n_words': 976}