<commit_before>#
# ------------------------------------------------------------
# Copyright (c) All rights reserved
# SiLab, Institute of Physics, University of Bonn
# ------------------------------------------------------------
#
# A transfer layer for SiTCP Ethernet. More information: http://sitcp.bbtech.co.jp
#

import logging
import socket
import select
import struct
import re
from array import array
from threading import Thread
from threading import RLock as Lock
from time import time

from basil.TL.SiTransferLayer import SiTransferLayer

logger = logging.getLogger(__name__)


class SiTcp(SiTransferLayer):
    '''SiTcp transport layer.transport

    UDP (RBCP) Header+Data

    Bit 7         Bit 0
    +-----------------+
    |  Ver.  |  Type  |
    +-----------------+
    |   CMD  |  FLAG  |
    +-----------------+
    |        ID       |
    +-----------------+
    |   Data Length   |
    +-----------------+
    | Address [31:24] |
    +-----------------+
    | Address [23:16] |
    +-----------------+
    | Address [15:8]  |
    +-----------------+
    | Address [7:0]   |
    +-----------------+
    |      Data 0     |
    +-----------------+
    |      Data 1     |
    +-----------------+
    |       ...       |
    +-----------------+
    |     Data N-1    |
    +-----------------+
    |      Data N     | (N max. 255)
    +-----------------+

    CMD Field

    +-----+------------+-------------+
    | BIT |    Name    | Description |
    +--------------------------------+
    |  3  |   Access   | Bus Access  |
    +--------------------------------+
    |  2  |    R/W     | 0:Wr,1:Read |
    +--------------------------------+
    |  1  |  Reserved  |   Always 0  |
    +--------------------------------+
    |  0  |  Reserved  |   Always 0  |
    +-----+------------+-------------+

    FALG Field

    +-----+------------+-------------+
    | BIT |    Name    | Description |
    +--------------------------------+
    |  3  |  REQ/ACK   | 0:Req,1:Ack |
    +--------------------------------+
    |  2  |  Reserved  |   Always 0  |
    +--------------------------------+
    |  1  |  Reserved  |   Always 0  |
    +--------------------------------+
    |  0  |    Error   |   0:Normal  |
    |     |            | 1:Bus Error |
    +-----+------------+-------------+


    TCP to BUS Header+Data

    Bit 7         Bit 0
    +-----------------+
    |   Length [7:0]  |
    +-----------------+
    |   Length [15:8] |
    +-----------------+
    | Address [7:0]   |
    +-----------------+
    | Address [15:8]  |
    +-----------------+
    | Address [23:16] |
    +-----------------+
    | Address [31:24] |
    +-----------------+
    |      Data 0     |
    +-----------------+
    |      Data 1     |
    +-----------------+
    |       ...       |
    +-----------------+
    |  Data Length-1  |
    +-----------------+
    |   Data Length   | (Length max. 65529)
    +-----------------+

    TCP to BUS reset sequence (in case of status invalid): 65535 * 0xff + 6 * 0x00

    '''
    # UDP(RBCP) interface
    RBCP_VER = 0xff
    RBCP_CMD_WR = 0x80
    RBCP_CMD_RD = 0xC0
    RBCP_MAX_SIZE = 255  # bytes

    UDP_TIMEOUT = 1.0
    UDP_RETRANSMIT_CNT = 3  # TODO

    BASE_DATA_TCP = 0x100000000
    BASE_FAKE_FIFO_TCP = 0x200000000  # above this read will return size of local TCP fifo (4 bytes)

    def __init__(self, conf):
        super(SiTcp, self).__init__(conf)
        self._sock_udp = None
        self._sock_tcp = None
        self._udp_lock = Lock()
        self._tcp_lock = Lock()
        self._tcp_readout_interval = 0.05
        self._tcp_readout_thread = None
        self._tcp_read_buff = None
        self.RBCP_ID = 0

    def reset(self):
        with self._tcp_lock:
            self._tcp_read_buff = array('B')

    def reset_fifo(self):
        with self._tcp_lock:
            fifo_size = self._get_tcp_data_size()
            fifo_int_size = (fifo_size - (fifo_size % 4)) / 4
            del_size = fifo_int_size * 4
            self._tcp_read_buff = self._tcp_read_buff[del_size:]

    def init(self):
        super(SiTcp, self).init()
        self.reset()
        if 'ip' not in self._init:
            raise ValueError('Parameter \'ip\' missing.')
        if 'udp_port' not in self._init:
            raise ValueError('Parameter \'udp_port\' missing.')
        self._sock_udp = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
        self._sock_udp.connect((self._init['ip'], self._init['udp_port']))
        # using select to monitor socket status, therefore the socket is set to blocking (default)
        self._sock_udp.setblocking(0)
        # start readout thread if TCP connection is set
        if 'tcp_connection' in self._init and self._init['tcp_connection']:
            if 'tcp_port' not in self._init:
                raise ValueError('Parameter \'tcp_port\' missing.')
            self._sock_tcp = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
            self._sock_tcp.connect((self._init['ip'], self._init['tcp_port']))
            # using select to monitor socket status, therefore the socket is set to blocking (default)
            self._sock_tcp.setblocking(0)
            self._tcp_readout_thread = Thread(target=self._tcp_readout, name='TcpReadoutThread', kwargs={})
            self._tcp_readout_thread.daemon = True  # exiting program even when thread is alive
            self._stop = False
            self._tcp_readout_thread.start()
            #self._reset_tcp_to_bus()
        else:
            self._sock_tcp = None

    def _write_single(self, addr, data):
        retry_write_cnt = 0
        while True:
            if self.RBCP_ID >= 255:
                self.RBCP_ID = 0
            else:
                self.RBCP_ID += 1
            request = array('B', struct.pack('>BBBBI', self.RBCP_VER, self.RBCP_CMD_WR, self.RBCP_ID, len(data), addr))
            request += data
            while True:
                rlist, _, _ = select.select([self._sock_udp], [], [], 0.0)
                if rlist:
                    # Read just enough for the header,
                    # remaining meassge data is lost.
                    ack = self._sock_udp.recv(3)
                    logger.warning('SiTcp:_write_single - Pending data before send - Message ID: current: %d, read: %d' % (self.RBCP_ID, ord(ack[2])))<commit_msg>ENH: always reset tcp_to_bus after initialization<commit_after>#
# ------------------------------------------------------------
# Copyright (c) All rights reserved
# SiLab, Institute of Physics, University of Bonn
# ------------------------------------------------------------
#
# A transfer layer for SiTCP Ethernet. More information: http://sitcp.bbtech.co.jp
#

import logging
import socket
import select
import struct
import re
from array import array
from threading import Thread
from threading import RLock as Lock
from time import time

from basil.TL.SiTransferLayer import SiTransferLayer

logger = logging.getLogger(__name__)


class SiTcp(SiTransferLayer):
    '''SiTcp transport layer.transport

    UDP (RBCP) Header+Data

    Bit 7         Bit 0
    +-----------------+
    |  Ver.  |  Type  |
    +-----------------+
    |   CMD  |  FLAG  |
    +-----------------+
    |        ID       |
    +-----------------+
    |   Data Length   |
    +-----------------+
    | Address [31:24] |
    +-----------------+
    | Address [23:16] |
    +-----------------+
    | Address [15:8]  |
    +-----------------+
    | Address [7:0]   |
    +-----------------+
    |      Data 0     |
    +-----------------+
    |      Data 1     |
    +-----------------+
    |       ...       |
    +-----------------+
    |     Data N-1    |
    +-----------------+
    |      Data N     | (N max. 255)
    +-----------------+

    CMD Field

    +-----+------------+-------------+
    | BIT |    Name    | Description |
    +--------------------------------+
    |  3  |   Access   | Bus Access  |
    +--------------------------------+
    |  2  |    R/W     | 0:Wr,1:Read |
    +--------------------------------+
    |  1  |  Reserved  |   Always 0  |
    +--------------------------------+
    |  0  |  Reserved  |   Always 0  |
    +-----+------------+-------------+

    FALG Field

    +-----+------------+-------------+
    | BIT |    Name    | Description |
    +--------------------------------+
    |  3  |  REQ/ACK   | 0:Req,1:Ack |
    +--------------------------------+
    |  2  |  Reserved  |   Always 0  |
    +--------------------------------+
    |  1  |  Reserved  |   Always 0  |
    +--------------------------------+
    |  0  |    Error   |   0:Normal  |
    |     |            | 1:Bus Error |
    +-----+------------+-------------+


    TCP to BUS Header+Data

    Bit 7         Bit 0
    +-----------------+
    |   Length [7:0]  |
    +-----------------+
    |   Length [15:8] |
    +-----------------+
    | Address [7:0]   |
    +-----------------+
    | Address [15:8]  |
    +-----------------+
    | Address [23:16] |
    +-----------------+
    | Address [31:24] |
    +-----------------+
    |      Data 0     |
    +-----------------+
    |      Data 1     |
    +-----------------+
    |       ...       |
    +-----------------+
    |  Data Length-1  |
    +-----------------+
    |   Data Length   | (Length max. 65529)
    +-----------------+

    TCP to BUS reset sequence (in case of status invalid): 65535 * 0xff + 6 * 0x00

    '''
    # UDP(RBCP) interface
    RBCP_VER = 0xff
    RBCP_CMD_WR = 0x80
    RBCP_CMD_RD = 0xC0
    RBCP_MAX_SIZE = 255  # bytes

    UDP_TIMEOUT = 1.0
    UDP_RETRANSMIT_CNT = 3  # TODO

    BASE_DATA_TCP = 0x100000000
    BASE_FAKE_FIFO_TCP = 0x200000000  # above this read will return size of local TCP fifo (4 bytes)

    def __init__(self, conf):
        super(SiTcp, self).__init__(conf)
        self._sock_udp = None
        self._sock_tcp = None
        self._udp_lock = Lock()
        self._tcp_lock = Lock()
        self._tcp_readout_interval = 0.05
        self._tcp_readout_thread = None
        self._tcp_read_buff = None
        self.RBCP_ID = 0

    def reset(self):
        with self._tcp_lock:
            self._tcp_read_buff = array('B')

    def reset_fifo(self):
        with self._tcp_lock:
            fifo_size = self._get_tcp_data_size()
            fifo_int_size = (fifo_size - (fifo_size % 4)) / 4
            del_size = fifo_int_size * 4
            self._tcp_read_buff = self._tcp_read_buff[del_size:]

    def init(self):
        super(SiTcp, self).init()
        self.reset()
        if 'ip' not in self._init:
            raise ValueError('Parameter \'ip\' missing.')
        if 'udp_port' not in self._init:
            raise ValueError('Parameter \'udp_port\' missing.')
        self._sock_udp = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
        self._sock_udp.connect((self._init['ip'], self._init['udp_port']))
        # using select to monitor socket status, therefore the socket is set to blocking (default)
        self._sock_udp.setblocking(0)
        # start readout thread if TCP connection is set
        if 'tcp_connection' in self._init and self._init['tcp_connection']:
            if 'tcp_port' not in self._init:
                raise ValueError('Parameter \'tcp_port\' missing.')
            self._sock_tcp = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
            self._sock_tcp.connect((self._init['ip'], self._init['tcp_port']))
            # using select to monitor socket status, therefore the socket is set to blocking (default)
            self._sock_tcp.setblocking(0)
            self._tcp_readout_thread = Thread(target=self._tcp_readout, name='TcpReadoutThread', kwargs={})
            self._tcp_readout_thread.daemon = True  # exiting program even when thread is alive
            self._stop = False
            self._tcp_readout_thread.start()
            if 'tcp_to_bus' in self._init and self._init['tcp_to_bus']:
                self._reset_tcp_to_bus()
        else:
            self._sock_tcp = None

    def _write_single(self, addr, data):
        retry_write_cnt = 0
        while True:
            if self.RBCP_ID >= 255:
                self.RBCP_ID = 0
            else:
                self.RBCP_ID += 1
            request = array('B', struct.pack('>BBBBI', self.RBCP_VER, self.RBCP_CMD_WR, self.RBCP_ID, len(data), addr))
            request += data
            while True:
                rlist, _, _ = select.select([self._sock_udp], [], [], 0.0)
                if rlist:
                    # Read just enough for the header,
                    # remaining meassge data is lost.
                    ack = self._sock_udp.recv(3)
                    logger.warning('SiTcp:_write_single - Pending data before send - Message ID: current: %d, read: %d' % (self.RBCP_ID, ord(ack[2])))
{'fraction_non_alphanumeric': 0.24425985424339786, 'fraction_numerical': 0.020217851265574797, 'mean_word_length': 1.170408163265306, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 4, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1255630', 'n_tokens_mistral': 4012, 'n_tokens_neox': 3704, 'n_words': 850}