<commit_before>#
#  Copyright (c) 2016-2017, The OpenThread Authors.
#  All rights reserved.
#
#  Licensed under the Apache License, Version 2.0 (the "License");
#  you may not use this file except in compliance with the License.
#  You may obtain a copy of the License at
#
#  http://www.apache.org/licenses/LICENSE-2.0
#
#  Unless required by applicable law or agreed to in writing, software
#  distributed under the License is distributed on an "AS IS" BASIS,
#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#  See the License for the specific language governing permissions and
#  limitations under the License.
#
"""
Module providing a Spienl coder / decoder class.
"""

import binascii
import time
import logging
import threading
import traceback
import queue
import importlib

from struct import pack
from struct import unpack
from collections import namedtuple
from collections import defaultdict

import ipaddress

import spinel.util as util
import spinel.config as CONFIG
from spinel.const import kThread
from spinel.const import SPINEL
from spinel.const import SPINEL_LAST_STATUS_MAP
from spinel.hdlc import Hdlc

FEATURE_USE_HDLC = 1
FEATURE_USE_SLACC = 1

TIMEOUT_PROP = 2

#=========================================
#   SpinelCodec
#=========================================

#  0: DATATYPE_NULL
#'.': DATATYPE_VOID: Empty data type. Used internally.
#'b': DATATYPE_BOOL: Boolean value. Encoded in 8-bits as either 0x00 or 0x01.
#                    All other values are illegal.
#'C': DATATYPE_UINT8: Unsigned 8-bit integer.
#'c': DATATYPE_INT8: Signed 8-bit integer.
#'S': DATATYPE_UINT16: Unsigned 16-bit integer. (Little-endian)
#'s': DATATYPE_INT16: Signed 16-bit integer. (Little-endian)
#'L': DATATYPE_UINT32: Unsigned 32-bit integer. (Little-endian)
#'l': DATATYPE_INT32: Signed 32-bit integer. (Little-endian)
#'i': DATATYPE_UINT_PACKED: Packed Unsigned Integer. (See section 7.2)
#'6': DATATYPE_IPv6ADDR: IPv6 Address. (Big-endian)
#'E': DATATYPE_EUI64: EUI-64 Address. (Big-endian)
#'e': DATATYPE_EUI48: EUI-48 Address. (Big-endian)
#'D': DATATYPE_DATA: Arbitrary Data. (See section 7.3)
#'d': DATATYPE_DATA_WLEN: Arbitrary Data with Prepended Length. (See section 7.3)
#'U': DATATYPE_UTF8: Zero-terminated UTF8-encoded string.
#'t': DATATYPE_STRUCT: Structured datatype. Compound type. Length prepended. (See section 7.4)
#'A': DATATYPE_ARRAY: Array of datatypes. Compound type. (See section 7.5)


class SpinelCodec(object):
    """ A general coder / decoder class for Spinel protocol. """

    @classmethod
    def parse_b(cls, payload):
        return unpack("<B", payload[:1])[0]

    @classmethod
    def parse_c(cls, payload):
        return unpack("<b", payload[:1])[0]

    @classmethod
    def parse_C(cls, payload):
        return unpack("<B", payload[:1])[0]

    @classmethod
    def parse_s(cls, payload):
        return unpack("<h", payload[:2])[0]

    @classmethod
    def parse_S(cls, payload):
        return unpack("<H", payload[:2])[0]

    @classmethod
    def parse_l(cls, payload):
        return unpack("<l", payload[:4])[0]

    @classmethod
    def parse_L(cls, payload):
        return unpack("<L", payload[:4])[0]

    @classmethod
    def parse_X(cls, payload):
        return unpack("<Q", payload[:8])[0]

    @classmethod
    def parse_6(cls, payload):
        return payload[:16]

    @classmethod
    def parse_E(cls, payload):
        return payload[:8]

    @classmethod
    def parse_e(cls, payload):
        return payload[:6]

    @classmethod
    def parse_U(cls, payload):
        payload = payload.decode('utf-8')
        nullchar = '\0'

        if payload.find(nullchar) >= 0:
            return payload[:payload.index(nullchar)]  # strip null
        else:
            return payload

    @classmethod
    def parse_D(cls, payload):
        return payload

    @classmethod
    def parse_d(cls, payload):
        return payload[2:2 + unpack("<H", payload[:2])[0]]

    @classmethod
    def parse_i(cls, payload):
        """ Decode EXI integer format. """
        value = 0
        value_len = 0
        value_mul = 1

        while value_len < 4:
            byte = payload[value_len]

            value += (byte & 0x7F) * value_mul
            if byte < 0x80:
                break
            value_mul *= 0x80
            value_len += 1

        return (value, value_len + 1)
<commit_msg>[codec] handle failures when decoding utf-8 (#114)<commit_after>#
#  Copyright (c) 2016-2017, The OpenThread Authors.
#  All rights reserved.
#
#  Licensed under the Apache License, Version 2.0 (the "License");
#  you may not use this file except in compliance with the License.
#  You may obtain a copy of the License at
#
#  http://www.apache.org/licenses/LICENSE-2.0
#
#  Unless required by applicable law or agreed to in writing, software
#  distributed under the License is distributed on an "AS IS" BASIS,
#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#  See the License for the specific language governing permissions and
#  limitations under the License.
#
"""
Module providing a Spienl coder / decoder class.
"""

import binascii
import time
import logging
import threading
import traceback
import queue
import importlib

from struct import pack
from struct import unpack
from collections import namedtuple
from collections import defaultdict

import ipaddress

import spinel.util as util
import spinel.config as CONFIG
from spinel.const import kThread
from spinel.const import SPINEL
from spinel.const import SPINEL_LAST_STATUS_MAP
from spinel.hdlc import Hdlc

FEATURE_USE_HDLC = 1
FEATURE_USE_SLACC = 1

TIMEOUT_PROP = 2

#=========================================
#   SpinelCodec
#=========================================

#  0: DATATYPE_NULL
#'.': DATATYPE_VOID: Empty data type. Used internally.
#'b': DATATYPE_BOOL: Boolean value. Encoded in 8-bits as either 0x00 or 0x01.
#                    All other values are illegal.
#'C': DATATYPE_UINT8: Unsigned 8-bit integer.
#'c': DATATYPE_INT8: Signed 8-bit integer.
#'S': DATATYPE_UINT16: Unsigned 16-bit integer. (Little-endian)
#'s': DATATYPE_INT16: Signed 16-bit integer. (Little-endian)
#'L': DATATYPE_UINT32: Unsigned 32-bit integer. (Little-endian)
#'l': DATATYPE_INT32: Signed 32-bit integer. (Little-endian)
#'i': DATATYPE_UINT_PACKED: Packed Unsigned Integer. (See section 7.2)
#'6': DATATYPE_IPv6ADDR: IPv6 Address. (Big-endian)
#'E': DATATYPE_EUI64: EUI-64 Address. (Big-endian)
#'e': DATATYPE_EUI48: EUI-48 Address. (Big-endian)
#'D': DATATYPE_DATA: Arbitrary Data. (See section 7.3)
#'d': DATATYPE_DATA_WLEN: Arbitrary Data with Prepended Length. (See section 7.3)
#'U': DATATYPE_UTF8: Zero-terminated UTF8-encoded string.
#'t': DATATYPE_STRUCT: Structured datatype. Compound type. Length prepended. (See section 7.4)
#'A': DATATYPE_ARRAY: Array of datatypes. Compound type. (See section 7.5)


class SpinelCodec(object):
    """ A general coder / decoder class for Spinel protocol. """

    @classmethod
    def parse_b(cls, payload):
        return unpack("<B", payload[:1])[0]

    @classmethod
    def parse_c(cls, payload):
        return unpack("<b", payload[:1])[0]

    @classmethod
    def parse_C(cls, payload):
        return unpack("<B", payload[:1])[0]

    @classmethod
    def parse_s(cls, payload):
        return unpack("<h", payload[:2])[0]

    @classmethod
    def parse_S(cls, payload):
        return unpack("<H", payload[:2])[0]

    @classmethod
    def parse_l(cls, payload):
        return unpack("<l", payload[:4])[0]

    @classmethod
    def parse_L(cls, payload):
        return unpack("<L", payload[:4])[0]

    @classmethod
    def parse_X(cls, payload):
        return unpack("<Q", payload[:8])[0]

    @classmethod
    def parse_6(cls, payload):
        return payload[:16]

    @classmethod
    def parse_E(cls, payload):
        return payload[:8]

    @classmethod
    def parse_e(cls, payload):
        return payload[:6]

    @classmethod
    def parse_U(cls, payload):
        nullchar = b'\0'
        return payload[:payload.index(nullchar)].decode('utf-8')  # strip null

    @classmethod
    def parse_D(cls, payload):
        return payload

    @classmethod
    def parse_d(cls, payload):
        return payload[2:2 + unpack("<H", payload[:2])[0]]

    @classmethod
    def parse_i(cls, payload):
        """ Decode EXI integer format. """
        value = 0
        value_len = 0
        value_mul = 1

        while value_len < 4:
            byte = payload[value_len]

            value += (byte & 0x7F) * value_mul
            if byte < 0x80:
                break
            value_mul *= 0x80
            value_len += 1

        return (value, value_len + 1)

{'fraction_non_alphanumeric': 0.1293360900528371, 'fraction_numerical': 0.025155065472088216, 'mean_word_length': 3.3103960396039604, 'pattern_counts': {'":': 0, '<': 25, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4844222', 'n_tokens_mistral': 2984, 'n_tokens_neox': 2714, 'n_words': 866}