<commit_before>#!/usr/bin/python
#
#  This file is part of the KNOT Project
#
#  Copyright (c) 2019, CESAR. All rights reserved.
#
#   This library is free software; you can redistribute it and/or
#   modify it under the terms of the GNU Lesser General Public
#   License as published by the Free Software Foundation; either
#   version 2.1 of the License, or (at your option) any later version.
#
#   This library is distributed in the hope that it will be useful,
#   but WITHOUT ANY WARRANTY; without even the implied warranty of
#   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
#   Lesser General Public License for more details.
#
#   You should have received a copy of the GNU Lesser General Public
#   License along with this library; if not, write to the Free Software
#   Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA

import socket
import struct
import json
import threading
import logging
import sys
import argparse

parser = argparse.ArgumentParser(description='Simulates a thing connection')
parser.add_argument('-f', '--file', dest="filename",
                    help="write credentials to FILE", default="storage.json",
                    type=str, metavar="FILE")
parser.add_argument('-i', '--id', dest="id",
                    help="KNoT id", default=0x0123456789abcdef,
                    type=int, metavar="ID")
parser.add_argument('-n', '--name', dest="name",
                    help="thing name", default='Test',
                    type=str, metavar="NAME")
parser.add_argument('-s', '--schema', dest="schema_file",
                    help="schema file", default='',
                    type=str, metavar="SCHEMA_FILE")
parser.add_argument('-d', '--data', dest="data_file",
                    help="data file", default='',
                    type=str, metavar="DATA_FILE")
parser.add_argument('-H', '--host', dest="host",
                    help="host to connect", default='localhost',
                    type=str, metavar="HOST")
parser.add_argument('-p', '--port', dest="port",
                    help="port to connect", default='8884',
                    type=int, choices=range(0x10000),metavar="PORT")
parser.add_argument('--debug', action="store_true",
                    help="show debug messages")
options = parser.parse_args()

if options.debug:
    logging.basicConfig(level=logging.DEBUG, stream=sys.stderr)
else:
    logging.basicConfig(level=logging.INFO, stream=sys.stderr)

THING_ID = options.id
THING_NAME = options.name
HOST = options.host
PORT = options.port
if options.schema_file:
    with open(options.schema_file) as fd:
        schemas = json.load(fd)['schema']
    if options.data_file:
        with open(options.data_file) as fd:
            datas = json.load(fd)['data']
    else:
        raise argparse.ArgumentTypeError('Missing data file')
else:
    schemas = [{
        "sensor_id": 253, "value_type":3,
        "unit":0, "type_id": 65521,
        "name": "Lamp Status"
    }]
    datas = [{
        'sensor_id': 253,
        'value': False
    }]
schemas_sents = []

# Definitions removed by knot-protocol

PROTO_REGISTER_REQ = 0x10
PROTO_REGISTER_RSP = 0x11
PROTO_UNREGISTER_REQ = 0x12
PROTO_UNREGISTER_RSP = 0x13
PROTO_AUTH_REQ = 0x14
PROTO_AUTH_RSP = 0x15

PROTO_PUSH_DATA_REQ = 0x20
PROTO_PUSH_DATA_RSP = 0x21
PROTO_POLL_DATA_REQ = 0x30
PROTO_POLL_DATA_RSP = 0x31

PROTO_SCHM_FRAG_REQ = 0x40
PROTO_SCHM_FRAG_RSP = 0x41
PROTO_SCHM_END_REQ = 0x42
PROTO_SCHM_END_RSP = 0x43

# Format for serialization

HEADER_FMT = 'BB'           # 2 uint8_t, (type, payload_len)
REGISTER_FMT = 'Q64s'       # uint64_t, char[64] (thing_id, name)
REGISTER_RSP_FMT = '37s40s' # char[37], char[40] (uuid, token)
AUTH_FMT = '37s40s'         # char[37], char[40] (uuid, token)
SCHEMA_FMT = 'BBBH22s' # 3 uint8_t, uint16_t, char[22] (sensor_id, value_type, unit, type_id, name)

# Util functions

def to_string(string):
    if not isinstance(string, bytes):
        string = string.encode('utf-8')
    return string

def set_interval(func, time, **kwargs):
    def func_wrapper():
        set_interval(func, time, **kwargs)
        func(**kwargs)
    t = threading.Timer(time, func_wrapper)
    t.daemon = True
    t.start()
    return t

def get_format_type(value):
    if isinstance(value, int):
        return 'i'
    elif isinstance(value, float):
        return 'f'
    elif isinstance(value, bool):
        return '?'
    return '16s'

def knot_proto_to_str(msg_type):
    return {
        0x10: 'REGISTER_REQUEST',
        0x11: 'REGISTER_RESPONSE',
        0x12: 'UNREGISTER_REQUEST',
        0x13: 'UNREGISTER_RESPONSE',
        0x14: 'AUTHENTICATE_REQUEST',
        0x15: 'AUTHENTICATE_RESPONSE',
        0x20: 'PUSH_DATA_REQUEST',
        0x21: 'PUSH_DATA_RESPONSE',
        0x30: 'POLL_DATA_REQUEST',
        0x31: 'POLL_DATA_RESPONSE',
        0x40: 'SCHEMA_FRAG_REQUEST',
        0x41: 'SCHEMA_FRAG_RESPONSE',
        0x42: 'SCHEMA_END_REQUEST',
        0x43: 'SCHEMA_END_RESPONSE'
    }.get(msg_type)

# Functions to send knot protocol messages

def send_knot_msg_register(sock, thing_id, name):
    logging.debug('[payload_len: %d] Sending knot_msg 0x%x (%s)', struct.calcsize(REGISTER_FMT),
                  PROTO_REGISTER_REQ, knot_proto_to_str(PROTO_REGISTER_REQ))
    header = struct.pack(HEADER_FMT, PROTO_REGISTER_REQ, struct.calcsize(REGISTER_FMT))
    msg = header + struct.pack('Q', thing_id) + struct.pack('64s', to_string(name))
    return sock.send(msg)

def send_knot_msg_auth(sock, uuid, token):
    logging.debug('[payload_len: %d] Sending knot_msg 0x%x (%s)', struct.calcsize(AUTH_FMT),
                  PROTO_AUTH_REQ, knot_proto_to_str(PROTO_AUTH_REQ))
    header = struct.pack(HEADER_FMT, PROTO_AUTH_REQ, struct.calcsize(AUTH_FMT))
    msg = header + struct.pack('77s', to_string(uuid + token))
    return sock.send(msg)

def send_knot_msg_schema(sock, sensor_id, value_type, unit, type_id, name):
    logging.debug('[payload_len: %d] Sending knot_msg 0x%x (%s)', struct.calcsize(SCHEMA_FMT),
                  PROTO_SCHM_FRAG_REQ, knot_proto_to_str(PROTO_SCHM_FRAG_REQ))
    logging.debug('Schema: %s', json.dumps({'sensor_id': sensor_id, 'value_type': value_type,
                                            'unit': unit, 'type_id': type_id, 'name': name}))
    header = struct.pack(HEADER_FMT, PROTO_SCHM_FRAG_REQ, struct.calcsize(SCHEMA_FMT))
    msg = header + struct.pack('BBB', sensor_id, value_type, unit)
    msg = msg + struct.pack('H', type_id)
    msg = msg + struct.pack('23s', to_string(name))
    return sock.send(msg)

def send_knot_msg_push_data(sock, sensor_id, value):
    logging.info('Sending data: sensor_id %s value %s', sensor_id, str(value))
    payload_len = struct.calcsize('B') + struct.calcsize(get_format_type(value))
    logging.debug('[payload_len:%d] Sending knot_msg 0x%x (%s)', payload_len, PROTO_PUSH_DATA_REQ,
                  knot_proto_to_str(PROTO_PUSH_DATA_REQ))
    header = struct.pack(HEADER_FMT, PROTO_PUSH_DATA_REQ, payload_len)
    msg = header + struct.pack('B', sensor_id)
    msg = msg + struct.pack(get_format_type(value), value)
    return sock.send(msg)

# Functions to handle knot protocol messages received

def handle_register(msg):<commit_msg>Send schema end if it is last sensor in test-conn<commit_after>#!/usr/bin/python
#
#  This file is part of the KNOT Project
#
#  Copyright (c) 2019, CESAR. All rights reserved.
#
#   This library is free software; you can redistribute it and/or
#   modify it under the terms of the GNU Lesser General Public
#   License as published by the Free Software Foundation; either
#   version 2.1 of the License, or (at your option) any later version.
#
#   This library is distributed in the hope that it will be useful,
#   but WITHOUT ANY WARRANTY; without even the implied warranty of
#   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
#   Lesser General Public License for more details.
#
#   You should have received a copy of the GNU Lesser General Public
#   License along with this library; if not, write to the Free Software
#   Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA

import socket
import struct
import json
import threading
import logging
import sys
import argparse

parser = argparse.ArgumentParser(description='Simulates a thing connection')
parser.add_argument('-f', '--file', dest="filename",
                    help="write credentials to FILE", default="storage.json",
                    type=str, metavar="FILE")
parser.add_argument('-i', '--id', dest="id",
                    help="KNoT id", default=0x0123456789abcdef,
                    type=int, metavar="ID")
parser.add_argument('-n', '--name', dest="name",
                    help="thing name", default='Test',
                    type=str, metavar="NAME")
parser.add_argument('-s', '--schema', dest="schema_file",
                    help="schema file", default='',
                    type=str, metavar="SCHEMA_FILE")
parser.add_argument('-d', '--data', dest="data_file",
                    help="data file", default='',
                    type=str, metavar="DATA_FILE")
parser.add_argument('-H', '--host', dest="host",
                    help="host to connect", default='localhost',
                    type=str, metavar="HOST")
parser.add_argument('-p', '--port', dest="port",
                    help="port to connect", default='8884',
                    type=int, choices=range(0x10000),metavar="PORT")
parser.add_argument('--debug', action="store_true",
                    help="show debug messages")
options = parser.parse_args()

if options.debug:
    logging.basicConfig(level=logging.DEBUG, stream=sys.stderr)
else:
    logging.basicConfig(level=logging.INFO, stream=sys.stderr)

THING_ID = options.id
THING_NAME = options.name
HOST = options.host
PORT = options.port
if options.schema_file:
    with open(options.schema_file) as fd:
        schemas = json.load(fd)['schema']
    if options.data_file:
        with open(options.data_file) as fd:
            datas = json.load(fd)['data']
    else:
        raise argparse.ArgumentTypeError('Missing data file')
else:
    schemas = [{
        "sensor_id": 253, "value_type":3,
        "unit":0, "type_id": 65521,
        "name": "Lamp Status"
    }]
    datas = [{
        'sensor_id': 253,
        'value': False
    }]
schemas_sents = []

# Definitions removed by knot-protocol

PROTO_REGISTER_REQ = 0x10
PROTO_REGISTER_RSP = 0x11
PROTO_UNREGISTER_REQ = 0x12
PROTO_UNREGISTER_RSP = 0x13
PROTO_AUTH_REQ = 0x14
PROTO_AUTH_RSP = 0x15

PROTO_PUSH_DATA_REQ = 0x20
PROTO_PUSH_DATA_RSP = 0x21
PROTO_POLL_DATA_REQ = 0x30
PROTO_POLL_DATA_RSP = 0x31

PROTO_SCHM_FRAG_REQ = 0x40
PROTO_SCHM_FRAG_RSP = 0x41
PROTO_SCHM_END_REQ = 0x42
PROTO_SCHM_END_RSP = 0x43

# Format for serialization

HEADER_FMT = 'BB'           # 2 uint8_t, (type, payload_len)
REGISTER_FMT = 'Q64s'       # uint64_t, char[64] (thing_id, name)
REGISTER_RSP_FMT = '37s40s' # char[37], char[40] (uuid, token)
AUTH_FMT = '<PASSWORD>'         # char[37], char[40] (uuid, token)
SCHEMA_FMT = 'BBBH22s' # 3 uint8_t, uint16_t, char[22] (sensor_id, value_type, unit, type_id, name)

# Util functions

def to_string(string):
    if not isinstance(string, bytes):
        string = string.encode('utf-8')
    return string

def set_interval(func, time, **kwargs):
    def func_wrapper():
        set_interval(func, time, **kwargs)
        func(**kwargs)
    t = threading.Timer(time, func_wrapper)
    t.daemon = True
    t.start()
    return t

def get_format_type(value):
    if isinstance(value, int):
        return 'i'
    elif isinstance(value, float):
        return 'f'
    elif isinstance(value, bool):
        return '?'
    return '16s'

def knot_proto_to_str(msg_type):
    return {
        0x10: 'REGISTER_REQUEST',
        0x11: 'REGISTER_RESPONSE',
        0x12: 'UNREGISTER_REQUEST',
        0x13: 'UNREGISTER_RESPONSE',
        0x14: 'AUTHENTICATE_REQUEST',
        0x15: 'AUTHENTICATE_RESPONSE',
        0x20: 'PUSH_DATA_REQUEST',
        0x21: 'PUSH_DATA_RESPONSE',
        0x30: 'POLL_DATA_REQUEST',
        0x31: 'POLL_DATA_RESPONSE',
        0x40: 'SCHEMA_FRAG_REQUEST',
        0x41: 'SCHEMA_FRAG_RESPONSE',
        0x42: 'SCHEMA_END_REQUEST',
        0x43: 'SCHEMA_END_RESPONSE'
    }.get(msg_type)

# Functions to send knot protocol messages

def send_knot_msg_register(sock, thing_id, name):
    logging.debug('[payload_len: %d] Sending knot_msg 0x%x (%s)', struct.calcsize(REGISTER_FMT),
                  PROTO_REGISTER_REQ, knot_proto_to_str(PROTO_REGISTER_REQ))
    header = struct.pack(HEADER_FMT, PROTO_REGISTER_REQ, struct.calcsize(REGISTER_FMT))
    msg = header + struct.pack('Q', thing_id) + struct.pack('64s', to_string(name))
    return sock.send(msg)

def send_knot_msg_auth(sock, uuid, token):
    logging.debug('[payload_len: %d] Sending knot_msg 0x%x (%s)', struct.calcsize(AUTH_FMT),
                  PROTO_AUTH_REQ, knot_proto_to_str(PROTO_AUTH_REQ))
    header = struct.pack(HEADER_FMT, PROTO_AUTH_REQ, struct.calcsize(AUTH_FMT))
    msg = header + struct.pack('77s', to_string(uuid + token))
    return sock.send(msg)

def send_knot_msg_schema(sock, sensor_id, value_type, unit, type_id, name):
    msg_type = PROTO_SCHM_FRAG_REQ if schemas else PROTO_SCHM_END_REQ
    logging.debug('[payload_len: %d] Sending knot_msg 0x%x (%s)', struct.calcsize(SCHEMA_FMT),
                  msg_type, knot_proto_to_str(msg_type))
    logging.debug('Schema: %s', json.dumps({'sensor_id': sensor_id, 'value_type': value_type,
                                            'unit': unit, 'type_id': type_id, 'name': name}))
    header = struct.pack(HEADER_FMT, msg_type, struct.calcsize(SCHEMA_FMT))
    msg = header + struct.pack('BBB', sensor_id, value_type, unit)
    msg = msg + struct.pack('H', type_id)
    msg = msg + struct.pack('23s', to_string(name))
    return sock.send(msg)

def send_knot_msg_push_data(sock, sensor_id, value):
    logging.info('Sending data: sensor_id %s value %s', sensor_id, str(value))
    payload_len = struct.calcsize('B') + struct.calcsize(get_format_type(value))
    logging.debug('[payload_len:%d] Sending knot_msg 0x%x (%s)', payload_len, PROTO_PUSH_DATA_REQ,
                  knot_proto_to_str(PROTO_PUSH_DATA_REQ))
    header = struct.pack(HEADER_FMT, PROTO_PUSH_DATA_REQ, payload_len)
    msg = header + struct.pack('B', sensor_id)
    msg = msg + struct.pack(get_format_type(value), value)
    return sock.send(msg)

# Functions to handle knot protocol messages received

def handle_register(msg):
{'fraction_non_alphanumeric': 0.12047032936808086, 'fraction_numerical': 0.0244791308533315, 'mean_word_length': 3.346682606096832, 'pattern_counts': {'":': 10, '<': 4, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1730362', 'n_tokens_mistral': 5154, 'n_tokens_neox': 4869, 'n_words': 1154}