<commit_before>#  Copyright 2014 Nokia Siemens Networks Oyj
#
#  Licensed under the Apache License, Version 2.0 (the "License");
#  you may not use this file except in compliance with the License.
#  You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
#  Unless required by applicable law or agreed to in writing, software
#  distributed under the License is distributed on an "AS IS" BASIS,
#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#  See the License for the specific language governing permissions and
#  limitations under the License.

from math import ceil
import re

from Rammbock.message import (Field, Union, Message, Header, List, Struct,
                              BinaryContainer, BinaryField, TBCDContainer,
                              Conditional, Bag)
from message_stream import MessageStream
from primitives import Length, Binary, TBCD, BagSize
from Rammbock.ordered_dict import OrderedDict
from Rammbock.binary_tools import (to_binary_string_of_length, to_bin,
                                   to_tbcd_value, to_tbcd_binary)
from Rammbock.condition_parser import ConditionParser
from Rammbock.logger import trace


class _Template(object):

    def __init__(self, name, parent):
        self.parent = parent
        self._fields = OrderedDict()
        self.name = name
        self._saved = False

    def _pretty_print_fields(self, fields):
        return ', '.join('%s:%s' % (key, value) for key, value in fields.items())

    def _mark_referenced_field(self, field):
        ref_field = self._get_field_recursive(field.length.field)
        if not ref_field:
            raise AssertionError('Length field %s unknown' % field.length.field)
        ref_field.referenced_later = True

    def add(self, field):
        if field.type == 'pdu':
            self._handle_pdu_field(field)
        if self._get_field(field.name):
            raise AssertionError("Duplicate field '%s' in '%s'" % (field.name, self._get_recursive_name()))
        if field.has_length and field.length.has_references:
            self._mark_referenced_field(field)
        self._fields[field.name] = field

    def _handle_pdu_field(self, field):
        raise AssertionError('PDU field not allowed')

    def _get_field(self, field_name):
        return self._fields.get(field_name)

    def _get_field_recursive(self, field_name):
        return self._get_field(field_name) or self.parent and self.parent._get_field_recursive(field_name)

    def _check_params_empty(self, message_fields, name):
        for key in message_fields.keys():
            if key.startswith('*'):
                message_fields.pop(key)
        if message_fields:
            raise AssertionError("Unknown fields in '%s': %s" %
                                 (self._get_recursive_name(), self._pretty_print_fields(message_fields)))

    def _get_recursive_name(self):
        return (self.parent._get_recursive_name() + "." if self.parent else '') + self.name

    def _encode_fields(self, struct, params, little_endian=False):
        for field in self._fields.values():
            encoded = field.encode(params, struct, little_endian=little_endian)
            # TODO: clean away this ugly hack that makes it possible to skip PDU
            # (now it is a 0 length place holder in header)
            if encoded:
                struct[field.name] = encoded
        self._check_params_empty(params, self.name)

    def decode(self, data, parent=None, name=None, little_endian=False):
        message = self._get_struct(name, parent)
        data_index = 0
        for field in self._fields.values():
            message[field.name] = field.decode(data[data_index:], message, little_endian=little_endian)
            data_index += len(message[field.name])
        return message

    def validate(self, message, message_fields):
        errors = []
        for field in self._fields.values():
            errors += field.validate(message, message_fields)
        self._check_params_empty(message_fields, self.name)
        return errors

    def _get_params_sub_tree(self, params, name=None):
        result = {'*': params['*']} if '*' in params else {}
        name = name or self.name
        for key in params.keys():
            prefix, _, ending = key.partition('.')
            if prefix == name:
                result[ending] = params.pop(key)
            elif prefix == '*' and ending:
                result[ending] = params[key]
        return result

    def _get_struct(self, name, parent):
        return None

    @property
    def is_saved(self):
        return self._saved


# TODO: Refactor the pdu to use the same dynamic length strategy as structs in encoding
class Protocol(_Template):

    def __init__(self, name, little_endian=False):
        _Template.__init__(self, name, None)
        self.pdu = None
        self.little_endian = little_endian
<commit_msg>Including the Rammbock instance<commit_after>#  Copyright 2014 Nokia Siemens Networks Oyj
#
#  Licensed under the Apache License, Version 2.0 (the "License");
#  you may not use this file except in compliance with the License.
#  You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
#  Unless required by applicable law or agreed to in writing, software
#  distributed under the License is distributed on an "AS IS" BASIS,
#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#  See the License for the specific language governing permissions and
#  limitations under the License.

from math import ceil
import re

from Rammbock.message import (Field, Union, Message, Header, List, Struct,
                              BinaryContainer, BinaryField, TBCDContainer,
                              Conditional, Bag)
from message_stream import MessageStream
from primitives import Length, Binary, TBCD, BagSize
from Rammbock.ordered_dict import OrderedDict
from Rammbock.binary_tools import (to_binary_string_of_length, to_bin,
                                   to_tbcd_value, to_tbcd_binary)
from Rammbock.condition_parser import ConditionParser
from Rammbock.logger import trace


class _Template(object):

    def __init__(self, name, parent):
        self.parent = parent
        self._fields = OrderedDict()
        self.name = name
        self._saved = False

    def _pretty_print_fields(self, fields):
        return ', '.join('%s:%s' % (key, value) for key, value in fields.items())

    def _mark_referenced_field(self, field):
        ref_field = self._get_field_recursive(field.length.field)
        if not ref_field:
            raise AssertionError('Length field %s unknown' % field.length.field)
        ref_field.referenced_later = True

    def add(self, field):
        if field.type == 'pdu':
            self._handle_pdu_field(field)
        if self._get_field(field.name):
            raise AssertionError("Duplicate field '%s' in '%s'" % (field.name, self._get_recursive_name()))
        if field.has_length and field.length.has_references:
            self._mark_referenced_field(field)
        self._fields[field.name] = field

    def _handle_pdu_field(self, field):
        raise AssertionError('PDU field not allowed')

    def _get_field(self, field_name):
        return self._fields.get(field_name)

    def _get_field_recursive(self, field_name):
        return self._get_field(field_name) or self.parent and self.parent._get_field_recursive(field_name)

    def _check_params_empty(self, message_fields, name):
        for key in message_fields.keys():
            if key.startswith('*'):
                message_fields.pop(key)
        if message_fields:
            raise AssertionError("Unknown fields in '%s': %s" %
                                 (self._get_recursive_name(), self._pretty_print_fields(message_fields)))

    def _get_recursive_name(self):
        return (self.parent._get_recursive_name() + "." if self.parent else '') + self.name

    def _encode_fields(self, struct, params, little_endian=False):
        for field in self._fields.values():
            encoded = field.encode(params, struct, little_endian=little_endian)
            # TODO: clean away this ugly hack that makes it possible to skip PDU
            # (now it is a 0 length place holder in header)
            if encoded:
                struct[field.name] = encoded
        self._check_params_empty(params, self.name)

    def decode(self, data, parent=None, name=None, little_endian=False):
        message = self._get_struct(name, parent)
        data_index = 0
        for field in self._fields.values():
            message[field.name] = field.decode(data[data_index:], message, little_endian=little_endian)
            data_index += len(message[field.name])
        return message

    def validate(self, message, message_fields):
        errors = []
        for field in self._fields.values():
            errors += field.validate(message, message_fields)
        self._check_params_empty(message_fields, self.name)
        return errors

    def _get_params_sub_tree(self, params, name=None):
        result = {'*': params['*']} if '*' in params else {}
        name = name or self.name
        for key in params.keys():
            prefix, _, ending = key.partition('.')
            if prefix == name:
                result[ending] = params.pop(key)
            elif prefix == '*' and ending:
                result[ending] = params[key]
        return result

    def _get_struct(self, name, parent):
        return None

    @property
    def is_saved(self):
        return self._saved


# TODO: Refactor the pdu to use the same dynamic length strategy as structs in encoding
class Protocol(_Template):

    def __init__(self, name, little_endian=False, library=None):
        _Template.__init__(self, name, None)
        self.pdu = None
        self.little_endian = little_endian
        self.library = library

{'fraction_non_alphanumeric': 0.09024218671490303, 'fraction_numerical': 0.0020098482564566375, 'mean_word_length': 2.7231575009352786, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4609246', 'n_tokens_mistral': 2855, 'n_tokens_neox': 2747, 'n_words': 818}