<commit_before>import logging
from bitarray import bitarray
import re
import os
import ast
import numpy as np
from collections import OrderedDict
import copy
import struct
import tables as tb
import datetime
from contextlib import contextmanager
from importlib import import_module
from operator import itemgetter

from pybar.analysis.RawDataConverter.data_struct import NameValue
from pybar.utils.utils import string_is_binary, flatten_iterable, iterable


flavors = ('fei4a', 'fei4b')


def bitarray_from_value(value, size=None, fmt='Q'):
    ba = bitarray(endian='little')
    ba.frombytes(struct.pack(fmt, value))
    if size is not None:
        if size > ba.length():
            ba.extend((size - ba.length()) * [0])
        else:
            ba = ba[:size]
    ba.reverse()
    return ba


class FEI4Register(object):
    def __init__(self, configuration_file=None, fe_type=None, chip_address=None, broadcast=False):
        '''

        Note:
        Chip ID: This 4-bit field consists of broadcast bit and chip address. The broadcast bit, the most significant one, if set, means that the command is broadcasted to all FE chips receiving the data stream.
        Chip address: The three least significant bits of the chip ID define the chip address and are compared with the geographical address of the chip (selected via wire bonding/jumpers).
        '''
        self.flavor = None
        if fe_type:
            self.init_fe_type(fe_type)

        self.broadcast = broadcast
        self.chip_address = None
        if chip_address is not None:
            self.set_chip_address(chip_address)

        self.configuration_file = fe_type
        if configuration_file:
            self.load_configuration(configuration_file)

        self.config_state = OrderedDict()

    def __repr__(self):
        return self.configuration_file

    def set_chip_address(self, chip_address):
        if 7 < chip_address < 0:
            raise ValueError('Chip address out of range: %i' % chip_address)
        self.chip_id_initialized = True
        self.chip_address = chip_address
        self.chip_id = chip_address
        if self.broadcast:
            self.chip_id += 0x8
        self.chip_id_bitarray = bitarray_from_value(value=self.chip_id, size=4, fmt='I')
        logging.info('Setting chip address to %d (broadcast bit %s)' % (self.chip_address, 'set' if self.broadcast else 'not set'))

    def init_fe_type(self, fe_type):
        self.flavor = None
        self.global_registers = {}
        self.pixel_registers = {}
        self.calibration_parameters = OrderedDict()
        self.miscellaneous = OrderedDict()
        self.commands = {}
        fei4_defines = import_module('pybar.fei4.fei4_defines')
        fe_type = getattr(fei4_defines, fe_type)
        if 'flavor' not in fe_type:
            raise ValueError('FEI4 flavor not defined')
        elif fe_type['flavor'] not in flavors:
            raise ValueError('Unknown FEI4 flavor: %s' % fe_type['flavor'])
        else:
            self.flavor = fe_type['flavor']
            logging.info('Initializing FEI4 registers (flavor: %s)' % self.flavor)
        for name, reg in fe_type['global_registers'].iteritems():
            address = reg.get('address')
            offset = reg.get('offset', 0)
            bitlength = reg.get('bitlength')
            addresses = range(address, address + (offset + bitlength + 16 - 1) / 16)
            littleendian = reg.get('littleendian', False)
            register_littleendian = reg.get('register_littleendian', False)
            value = reg.get('value', 0)
            if not 0 <= value < 2 ** bitlength:
                raise ValueError("Global register %s: value exceeds limits" % (name,))
            readonly = reg.get('readonly', False)
            description = reg.get('description', '')
            self.global_registers[name] = dict(name=name, address=address, offset=offset, bitlength=bitlength, addresses=addresses, littleendian=littleendian, register_littleendian=register_littleendian, value=value, readonly=readonly, description=description)
        for name, reg in fe_type['pixel_registers'].iteritems():
            pxstrobe = reg.get('pxstrobe')
            bitlength = reg.get('bitlength')
            if bitlength > 8:
                raise Exception('Pixel register %s: up to 8 bits supported' % (name,))  # numpy array dtype is uint8
            littleendian = reg.get('littleendian', False)
            if not 0 <= reg.get('value', 0) < 2 ** bitlength:
                raise ValueError("Global register %s: value exceeds limits" % (name,))
            value = np.full((80, 336), reg.get('value', 0), dtype=np.uint8)
            description = reg.get('description', '')
            self.pixel_registers[name] = dict(name=name, pxstrobe=pxstrobe, bitlength=bitlength, littleendian=littleendian, value=value, description=description)
        for name, command in fe_type['commands'].iteritems():
            bitlength = command.get('bitlength')
            description = command.get('description', '')
            if 'bitstream' in command:
                bitstream = command.get('bitstream')
                self.commands[name] = dict(name=name, bitstream=bitstream, bitlength=bitlength, description=description)
            else:
                self.commands[name] = dict(name=name, bitlength=bitlength, description=description)
        self.calibration_parameters = fe_type['calibration_parameters'].copy()

    def is_chip_flavor(self, chip_flavor):
        if chip_flavor in flavors:
            if chip_flavor == self.flavor:
                return True
            else:
                return False
        else:
            raise ValueError('Unknown FEI4 flavor: %s' % chip_flavor)

    @property
    def chip_flavor(self):
        return self.flavor

    @property
    def fei4a(self):
        return True if self.flavor == 'fei4a' else False

    @property
    def fei4b(self):
        return True if self.flavor == 'fei4b' else False

    def load_configuration(self, configuration_file):
        '''Loading configuration

        Parameters
        ----------
        configuration_file : string
            Path to the configuration file (text or HDF5 file).
        '''
        if os.path.isfile(configuration_file):
            if not isinstance(configuration_file, tb.file.File) and os.path.splitext(configuration_file)[1].strip().lower() != ".h5":
                self._load_configuration_from_text_file(configuration_file)
            else:
                self._load_configuration_from_hdf5(configuration_file)
        else:
            raise ValueError('Cannot find configuration file specified: %s' % configuration_file)

    def save_configuration(self, configuration_file):
        '''Saving configuration

        Parameters
        ----------
        configuration_file : string
            Filename of the configuration file.
        '''
        if not isinstance(configuration_file, tb.file.File) and os.path.splitext(configuration_file)[1].strip().lower() != ".h5":
            return self._save_configuration_to_text_file(configuration_file)
        else:
            return self._save_configuration_to_hdf5(configuration_file)

    def _load_configuration_from_text_file(self, configuration_file):
        '''Loading configuration from text files

        Parameters
        ----------
        configuration_file : string
            Full path (directory and filename) of the configuration file. If name is not given, reload configuration from file.
        '''
        logging.info("Loading configuration: %s" % configuration_file)
        self.configuration_file = configuration_file
        with open(configuration_file, 'r') as f:
            f.seek(0)
            config_dict = {}
            for line in f.readlines():
                line = line.partition('#')[0].strip()
                if not line:
                    continue
                parts = re.split(r'\s*[=]\s*|\s+', line)
                if parts[0] in config_dict:
                    logging.warning('Item %s in configuration file exists more than once' % parts[0])
                try:
                    config_dict[parts[0]] = ast.literal_eval(parts[1])
                except SyntaxError:
                    config_dict[parts[0]] = parts[1].strip()
                except ValueError:
                    config_dict[parts[0]] = parts[1].strip()

        if 'Flavor' in config_dict:
            flavor = config_dict.pop('Flavor')
            if self.flavor:
                pass
            else:
                self.init_fe_type(flavor)
        else:
            if self.flavor:
                pass
            else:
                raise ValueError('Flavor not specified')
        if 'Chip_ID' in config_dict:
            chip_id = config_dict.pop('Chip_ID')
            if self.chip_address:
                pass
            else:
                self.broadcast = True if chip_id & 0x8 else False
                self.set_chip_address(chip_id & 0x7)
        elif 'Chip_Address' in config_dict:
            chip_address = config_dict.pop('Chip_Address')
            if self.chip_address:
                pass
            else:
                self.set_chip_address(chip_address)
        else:
            if self.chip_id_initialized:
                pass
            else:
                raise ValueError('Chip address not specified')
        global_registers_configured = []
        pixel_registers_configured = []
        for key in config_dict.keys():
            value = config_dict.pop(key)
            if key in self.global_registers:
                self.set_global_register_value(key, value)
                global_registers_configured.append(key)
            elif key in self.pixel_registers:
                self.set_pixel_register_value(key, value)
                pixel_registers_configured.append(key)
            elif key in self.calibration_parameters:
                self.calibration_parameters[key] = value
            else:
                self.miscellaneous[key] = value

        global_registers = self.get_global_register_attributes('name', readonly=False)
        pixel_registers = self.pixel_registers.keys()
        global_registers_not_configured = set(global_registers).difference(global_registers_configured)
        pixel_registers_not_configured = set(pixel_registers).difference(pixel_registers_configured)
        if global_registers_not_configured:
            logging.warning("Following global register(s) not configured: {}".format(', '.join('\'' + reg + '\'' for reg in global_registers_not_configured)))
        if pixel_registers_not_configured:
            logging.warning("Following pixel register(s) not configured: {}".format(', '.join('\'' + reg + '\'' for reg in pixel_registers_not_configured)))
        if self.miscellaneous:
            logging.warning("Found following unknown parameter(s): {}".format(', '.join('\'' + parameter + '\'' for parameter in self.miscellaneous.iterkeys())))

    def _save_configuration_to_text_file(self, configuration_file):
        '''Saving configuration to text files

        Parameters
        ----------
        configuration_file : string
            Filename of the configuration file.
        '''
        configuration_path, filename = os.path.split(configuration_file)
        if os.path.split(configuration_path)[1] == 'configs':
            configuration_path = os.path.split(configuration_path)[0]
        filename = os.path.splitext(filename)[0].strip()
        self.configuration_file = os.path.join(os.path.join(configuration_path, 'configs'), filename + ".cfg")
        if os.path.isfile(self.configuration_file):
            logging.warning("Overwriting configuration: %s" % self.configuration_file)
        else:
            logging.info("Saving configuration: %s" % self.configuration_file)
        pixel_reg_dict = {}
        for path in ["tdacs", "fdacs", "masks", "configs"]:
            configuration_file_path = os.path.join(configuration_path, path)
            if not os.path.exists(configuration_file_path):
                os.makedirs(configuration_file_path)
            if path == "tdacs":
                dac = self.get_pixel_register_objects(name="TDAC")[0]
                dac_config_path = os.path.join(configuration_file_path, "_".join([dac['name'].lower(), filename]) + ".dat")
                self._write_pixel_dac_config(dac_config_path, dac['value'])
                pixel_reg_dict[dac['name']] = os.path.relpath(dac_config_path, os.path.dirname(self.configuration_file))
            elif path == "fdacs":
                dac = self.get_pixel_register_objects(name="FDAC")[0]
                dac_config_path = os.path.join(configuration_file_path, "_".join([dac['name'].lower(), filename]) + ".dat")
                self._write_pixel_dac_config(dac_config_path, dac['value'])
                pixel_reg_dict[dac['name']] = os.path.relpath(dac_config_path, os.path.dirname(self.configuration_file))
            elif path == "masks":
                masks = self.get_pixel_register_objects(bitlength=1)
                for mask in masks:
                    dac_config_path = os.path.join(configuration_file_path, "_".join([mask['name'].lower(), filename]) + ".dat")
                    self._write_pixel_mask_config(dac_config_path, mask['value'])
                    pixel_reg_dict[mask['name']] = os.path.relpath(dac_config_path, os.path.dirname(self.configuration_file))
            elif path == "configs":
                with open(self.configuration_file, 'w') as f:
                    lines = []
                    lines.append("# FEI4 Flavor\n")
                    lines.append('%s %s\n' % ('Flavor', self.flavor))
                    lines.append("\n# FEI4 Chip ID\n")
                    lines.append('%s %d\n' % ('Chip_ID', self.chip_id))
                    lines.append("\n# FEI4 Global Registers\n")
                    global_regs = self.get_global_register_objects(readonly=False)
                    for global_reg in sorted(global_regs, key=itemgetter('name')):
                        lines.append('%s %d\n' % (global_reg['name'], global_reg['value']))
                    lines.append("\n# FEI4 Pixel Registers\n")
                    for key in sorted(pixel_reg_dict):
                        lines.append('%s %s\n' % (key, pixel_reg_dict[key]))
                    if self.miscellaneous:
                        lines.append("\n# Miscellaneous\n")
                        for key, value in self.miscellaneous.iteritems():
                            lines.append('%s %s\n' % (key, value))
                    f.writelines(lines)

    def _load_configuration_from_hdf5(self, configuration_file, node=''):
        '''Loading configuration from HDF5 file

        Parameters
        ----------
        configuration_file : string, file
            Filename of the HDF5 configuration file or file object.
        node : string
            Additional identifier (subgroup). Useful when more than one configuration is stored inside a HDF5 file.
        '''
        def load_conf():
            logging.info("Loading configuration: %s" % h5_file.filename)
            self.configuration_file = h5_file.filename
            if node:
                configuration_group = h5_file.root.configuration.node
            else:
                configuration_group = h5_file.root.configuration

            # miscellaneous
            for row in configuration_group.miscellaneous:
                name = row['name']
                try:
                    value = ast.literal_eval(row['value'])
                except ValueError:
                    value = row['value']
                if name == 'Flavor':
                    if self.flavor:
                        pass
                    else:
                        self.init_fe_type(value)
                elif name == 'Chip_ID':
                    if self.chip_address:
                        pass
                    else:
                        self.broadcast = True if value & 0x8 else False
                        self.set_chip_address(value & 0x7)
                elif name == 'Chip_Address':
                    if self.chip_address:
                        pass
                    else:
                        self.set_chip_address(value)
                else:
                    self.miscellaneous[name] = value

            if self.flavor:
                pass
            else:
                raise ValueError('Flavor not specified')

            if self.chip_id_initialized:
                pass
            else:
                raise ValueError('Chip address not specified')

            # calibration parameters
            for row in configuration_group.calibration_parameters:
                name = row['name']
                value = row['value']
                self.calibration_parameters[name] = ast.literal_eval(value)

            # global
            for row in configuration_group.global_register:
                name = row['name']
                value = row['value']
                self.set_global_register_value(name, ast.literal_eval(value))

            # pixels
            for pixel_reg in h5_file.iter_nodes(configuration_group, 'CArray'):  # ['Enable', 'TDAC', 'C_High', 'C_Low', 'Imon', 'FDAC', 'EnableDigInj']:
                if pixel_reg.name in self.pixel_registers:
                    self.set_pixel_register_value(pixel_reg.name, np.asarray(pixel_reg).T)  # np.asarray(h5_file.get_node(configuration_group, name=pixel_reg)).T

        if isinstance(configuration_file, tb.file.File):
            h5_file = configuration_file
            load_conf()
        else:
            with tb.open_file(configuration_file, mode="r", title='') as h5_file:
                load_conf()

    def _save_configuration_to_hdf5(self, configuration_file, name=''):
        '''Saving configuration to HDF5 file

        Parameters
        ----------
        configuration_file : string, file
            Filename of the HDF5 configuration file or file object.
        name : string
            Additional identifier (subgroup). Useful when storing more than one configuration inside a HDF5 file.
        '''
        def save_conf():
            logging.info("Saving configuration: %s" % h5_file.filename)
            self.configuration_file = h5_file.filename
            try:
                configuration_group = h5_file.create_group(h5_file.root, "configuration")
            except tb.NodeError:
                configuration_group = h5_file.root.configuration
            if name:
                try:
                    configuration_group = h5_file.create_group(configuration_group, name)
                except tb.NodeError:
                    configuration_group = h5_file.root.configuration.name

            # calibration_parameters
            try:
                h5_file.remove_node(configuration_group, name='calibration_parameters')
            except tb.NodeError:
                pass
            calibration_data_table = h5_file.create_table(configuration_group, name='calibration_parameters', description=NameValue, title='calibration_parameters')
            calibration_data_row = calibration_data_table.row
            for key, value in self.calibration_parameters.iteritems():
                calibration_data_row['name'] = key
                calibration_data_row['value'] = value
                calibration_data_row.append()
            calibration_data_table.flush()

            # miscellaneous
            try:
                h5_file.remove_node(configuration_group, name='miscellaneous')
            except tb.NodeError:
                pass
            miscellaneous_data_table = h5_file.create_table(configuration_group, name='miscellaneous', description=NameValue, title='miscellaneous')
            miscellaneous_data_row = miscellaneous_data_table.row
            miscellaneous_data_row['name'] = 'Flavor'
            miscellaneous_data_row['value'] = self.flavor
            miscellaneous_data_row.append()
            miscellaneous_data_row['name'] = 'Chip_ID'
            miscellaneous_data_row['value'] = self.chip_id
            miscellaneous_data_row.append()
            for key, value in self.miscellaneous.iteritems():
                miscellaneous_data_row['name'] = key
                miscellaneous_data_row['value'] = value
                miscellaneous_data_row.append()
            miscellaneous_data_table.flush()

            # global
            try:
                h5_file.remove_node(configuration_group, name='global_register')
            except tb.NodeError:
                pass
            global_data_table = h5_file.create_table(configuration_group, name='global_register', description=NameValue, title='global_register')
            global_data_table_row = global_data_table.row
            global_regs = self.get_global_register_objects(readonly=False)
            for global_reg in sorted(global_regs, key=itemgetter('name')):
                global_data_table_row['name'] = global_reg['name']
                global_data_table_row['value'] = global_reg['value']  # TODO: some function that converts to bin, hex
                global_data_table_row.append()
            global_data_table.flush()

            # pixel
            for pixel_reg in self.pixel_registers.itervalues():
                try:
                    h5_file.remove_node(configuration_group, name=pixel_reg['name'])
                except tb.NodeError:
                    pass
                data = pixel_reg['value'].T
                atom = tb.Atom.from_dtype(data.dtype)
                ds = h5_file.createCArray(configuration_group, name=pixel_reg['name'], atom=atom, shape=data.shape, title=pixel_reg['name'])
                ds[:] = data

        if isinstance(configuration_file, tb.file.File):
            h5_file = configuration_file
            save_conf()
        else:
            with tb.open_file(configuration_file, mode="a", title='') as h5_file:
                save_conf()

    def parse_parameters(self, parameters):
        with open(self.configuration_file, 'r') as f:
            for line in f.readlines():
                key_value = re.split(r'\s*[=]\s*|\s+', line)
                if key_value[0] in parameters:
                    try:
                        parameters[key_value[0]] = ast.literal_eval(key_value[1].strip())
                    except SyntaxError:  # for comma separated values, e.g. lists
                        parameters[key_value[0]] = ast.literal_eval(line[len(key_value[0]):].strip())
                    except ValueError:
                        parameters[key_value[0]] = key_value[1].strip().lower()

    def write_parameters(self, parameters, title=None):
        with open(self.configuration_file, 'a') as f:
            lines = []
            if title:
                lines.append("# %s\n" % title)
            for key, value in parameters.iteritems():
                lines.append('%s %s\n' % (key, str(value)))
            lines.append("\n")
            f.writelines(lines)

    def _parse_pixel_mask_config(self, filename):
        mask = np.empty((80, 336), dtype=np.uint8)
        with open(filename, 'r') as f:
            row = 0
            for line in f.readlines():
                line = line.split()
                if len(line) == 0 or line[0][0] == '#':
                    continue
                try:
                    int(line[0])
                except ValueError:
                    line = ''.join(line).translate(None, '_-')
                else:
                    line = ''.join(line[1:]).translate(None, '_-')
                if len(line) != 80:
                    raise ValueError('Dimension of column')
                # for col, value in enumerate(line):
                #    mask[col][row] = value
                mask[:, row] = list(line)
                row += 1
            if row != 336:
                raise ValueError('Dimension of row')
        return mask

    def _write_pixel_mask_config(self, filename, value):
        with open(filename, 'w') as f:
            seq = []
            seq.append("###  1     6     11    16     21    26     31    36     41    46     51    56     61    66     71    76\n")
            seq.append("\n".join([(repr(row + 1).rjust(3) + "  ") + "  ".join(["-".join(["".join([repr(value[col, row]) for col in range(col_fine, col_fine + 5)]) for col_fine in range(col_coarse, col_coarse + 10, 5)]) for col_coarse in range(0, 80, 10)]) for row in range(336)]))
            seq.append("\n")
            f.writelines(seq)

    def _parse_pixel_dac_config(self, filename):
        mask = np.empty((80, 336), dtype=np.uint8)
        with open(filename, 'r') as f:
            row = 0
            read_line = 0
            for line in f.readlines():
                line = line.split()
                if len(line) == 0 or line[0][0] == '#':
                    continue
                try:
                    int(line[0])
                except ValueError:
                    line = line[1:]
                else:
                    pass  # nothing to do
                if len(line) != 40:
                    raise ValueError('Dimension of column')
                if read_line % 2 == 0:
                    mask[:40, row] = line
                else:
                    mask[40:, row] = line
                    row += 1
                read_line += 1
            if row != 336:
                raise ValueError('Dimension of row')
        return mask

    def _write_pixel_dac_config(self, filename, value):
        with open(filename, 'w') as f:
            seq = []
            seq.append("###    1  2  3  4  5  6  7  8  9 10   11 12 13 14 15 16 17 18 19 20   21 22 23 24 25 26 27 28 29 30   31 32 33 34 35 36 37 38 39 40\n")
            seq.append("###   41 42 43 44 45 46 47 48 49 50   51 52 53 54 55 56 57 58 59 60   61 62 63 64 65 66 67 68 69 70   71 72 73 74 75 76 77 78 79 80\n")
            seq.append("\n".join(["\n".join([((repr(row + 1).rjust(3) + ("a" if col_coarse == 0 else "b") + "  ") + "   ".join([" ".join([repr(value[col, row]).rjust(2) for col in range(col_fine, col_fine + 10)]) for col_fine in range(col_coarse, col_coarse + 40, 10)])) for col_coarse in range(0, 80, 40)]) for row in range(336)]))
            seq.append("\n")
            f.writelines(seq)

    '''
    TODO:
    for the following functions use
    filter(function, iterable).

    Make new generic function that uses filter.

    Use next(iterator[, default]).
    '''

    def set_global_register_value(self, name, value):
        if self.global_registers[name]['readonly']:
            raise ValueError('Global register %s: register is read-only' % name)
        value = long(str(value), 0)  # value is decimal string or number or BitVector
        if not 0 <= self.global_registers[name]['value'] < 2 ** self.global_registers[name]['bitlength']:
            raise ValueError('Global register %s: value exceeds limits' % name)
        self.global_registers[name]['value'] = value

    def get_global_register_value(self, name):
        return self.global_registers[name]['value']

    def set_pixel_register_value(self, name, value):
        try:  # value is decimal string or number or array
            self.pixel_registers[name]['value'][:, :] = value
        except ValueError:  # value is path to pixel config
            if self.pixel_registers[name]['bitlength'] == 1:  # pixel mask
                if value[0] == "~" or value[0] == "!":
                    reg_value = self._parse_pixel_mask_config(os.path.join(os.path.dirname(self.configuration_file), value[1:]))
                    inverted_mask = np.ones(shape=(80, 336), dtype=np.dtype('>u1'))
                    inverted_mask[reg_value >= 1] = 0
                    self.pixel_registers[name]['value'][:, :] = inverted_mask
                else:
                    self.pixel_registers[name]['value'][:, :] = self._parse_pixel_mask_config(os.path.join(os.path.dirname(self.configuration_file), value))
            else:  # pixel dac
                self.pixel_registers[name]['value'][:, :] = self._parse_pixel_dac_config(os.path.join(os.path.dirname(self.configuration_file), value))
        if (self.pixel_registers[name]['value'] >= 2 ** self.pixel_registers[name]['bitlength']).any() or (self.pixel_registers[name]['value'] < 0).any():
            raise ValueError("Pixel register %s: value exceeds limits" % name)

    def get_pixel_register_value(self, name):
        return self.pixel_registers[name]['value'].copy()

    def get_commands(self, command_name, **kwargs):
        """get fe_command from command name and keyword arguments

        wrapper for build_commands()
        implements FEI4 specific behavior

        """
        chip_id = kwargs.pop("ChipID", self.chip_id_bitarray)
        commands = []
        if command_name == "zeros":
            if "length" in kwargs:
                bv = bitarray(kwargs["length"], endian='little')  # all bits to zero
            elif "mask_steps" in kwargs:<commit_msg>MAINT: make some helper methods non class methods to be able to use them externally <commit_after>import logging
from bitarray import bitarray
import re
import os
import ast
import numpy as np
from collections import OrderedDict
import copy
import struct
import tables as tb
import datetime
from contextlib import contextmanager
from importlib import import_module
from operator import itemgetter

from pybar.analysis.RawDataConverter.data_struct import NameValue
from pybar.utils.utils import string_is_binary, flatten_iterable, iterable


def parse_pixel_mask_config(filename):
    mask = np.empty((80, 336), dtype=np.uint8)
    with open(filename, 'r') as f:
        row = 0
        for line in f.readlines():
            line = line.split()
            if len(line) == 0 or line[0][0] == '#':
                continue
            try:
                int(line[0])
            except ValueError:
                line = ''.join(line).translate(None, '_-')
            else:
                line = ''.join(line[1:]).translate(None, '_-')
            if len(line) != 80:
                raise ValueError('Dimension of column')
            # for col, value in enumerate(line):
            #    mask[col][row] = value
            mask[:, row] = list(line)
            row += 1
        if row != 336:
            raise ValueError('Dimension of row')
    return mask


def write_pixel_mask_config(filename, value):
    with open(filename, 'w') as f:
        seq = []
        seq.append("###  1     6     11    16     21    26     31    36     41    46     51    56     61    66     71    76\n")
        seq.append("\n".join([(repr(row + 1).rjust(3) + "  ") + "  ".join(["-".join(["".join([repr(value[col, row]) for col in range(col_fine, col_fine + 5)]) for col_fine in range(col_coarse, col_coarse + 10, 5)]) for col_coarse in range(0, 80, 10)]) for row in range(336)]))
        seq.append("\n")
        f.writelines(seq)


def parse_pixel_dac_config(filename):
    mask = np.empty((80, 336), dtype=np.uint8)
    with open(filename, 'r') as f:
        row = 0
        read_line = 0
        for line in f.readlines():
            line = line.split()
            if len(line) == 0 or line[0][0] == '#':
                continue
            try:
                int(line[0])
            except ValueError:
                line = line[1:]
            else:
                pass  # nothing to do
            if len(line) != 40:
                raise ValueError('Dimension of column')
            if read_line % 2 == 0:
                mask[:40, row] = line
            else:
                mask[40:, row] = line
                row += 1
            read_line += 1
        if row != 336:
            raise ValueError('Dimension of row')
    return mask


def write_pixel_dac_config(filename, value):
    with open(filename, 'w') as f:
        seq = []
        seq.append("###    1  2  3  4  5  6  7  8  9 10   11 12 13 14 15 16 17 18 19 20   21 22 23 24 25 26 27 28 29 30   31 32 33 34 35 36 37 38 39 40\n")
        seq.append("###   41 42 43 44 45 46 47 48 49 50   51 52 53 54 55 56 57 58 59 60   61 62 63 64 65 66 67 68 69 70   71 72 73 74 75 76 77 78 79 80\n")
        seq.append("\n".join(["\n".join([((repr(row + 1).rjust(3) + ("a" if col_coarse == 0 else "b") + "  ") + "   ".join([" ".join([repr(value[col, row]).rjust(2) for col in range(col_fine, col_fine + 10)]) for col_fine in range(col_coarse, col_coarse + 40, 10)])) for col_coarse in range(0, 80, 40)]) for row in range(336)]))
        seq.append("\n")
        f.writelines(seq)



flavors = ('fei4a', 'fei4b')


def bitarray_from_value(value, size=None, fmt='Q'):
    ba = bitarray(endian='little')
    ba.frombytes(struct.pack(fmt, value))
    if size is not None:
        if size > ba.length():
            ba.extend((size - ba.length()) * [0])
        else:
            ba = ba[:size]
    ba.reverse()
    return ba


class FEI4Register(object):
    def __init__(self, configuration_file=None, fe_type=None, chip_address=None, broadcast=False):
        '''

        Note:
        Chip ID: This 4-bit field consists of broadcast bit and chip address. The broadcast bit, the most significant one, if set, means that the command is broadcasted to all FE chips receiving the data stream.
        Chip address: The three least significant bits of the chip ID define the chip address and are compared with the geographical address of the chip (selected via wire bonding/jumpers).
        '''
        self.flavor = None
        if fe_type:
            self.init_fe_type(fe_type)

        self.broadcast = broadcast
        self.chip_address = None
        if chip_address is not None:
            self.set_chip_address(chip_address)

        self.configuration_file = fe_type
        if configuration_file:
            self.load_configuration(configuration_file)

        self.config_state = OrderedDict()

    def __repr__(self):
        return self.configuration_file

    def set_chip_address(self, chip_address):
        if 7 < chip_address < 0:
            raise ValueError('Chip address out of range: %i' % chip_address)
        self.chip_id_initialized = True
        self.chip_address = chip_address
        self.chip_id = chip_address
        if self.broadcast:
            self.chip_id += 0x8
        self.chip_id_bitarray = bitarray_from_value(value=self.chip_id, size=4, fmt='I')
        logging.info('Setting chip address to %d (broadcast bit %s)' % (self.chip_address, 'set' if self.broadcast else 'not set'))

    def init_fe_type(self, fe_type):
        self.flavor = None
        self.global_registers = {}
        self.pixel_registers = {}
        self.calibration_parameters = OrderedDict()
        self.miscellaneous = OrderedDict()
        self.commands = {}
        fei4_defines = import_module('pybar.fei4.fei4_defines')
        fe_type = getattr(fei4_defines, fe_type)
        if 'flavor' not in fe_type:
            raise ValueError('FEI4 flavor not defined')
        elif fe_type['flavor'] not in flavors:
            raise ValueError('Unknown FEI4 flavor: %s' % fe_type['flavor'])
        else:
            self.flavor = fe_type['flavor']
            logging.info('Initializing FEI4 registers (flavor: %s)' % self.flavor)
        for name, reg in fe_type['global_registers'].iteritems():
            address = reg.get('address')
            offset = reg.get('offset', 0)
            bitlength = reg.get('bitlength')
            addresses = range(address, address + (offset + bitlength + 16 - 1) / 16)
            littleendian = reg.get('littleendian', False)
            register_littleendian = reg.get('register_littleendian', False)
            value = reg.get('value', 0)
            if not 0 <= value < 2 ** bitlength:
                raise ValueError("Global register %s: value exceeds limits" % (name,))
            readonly = reg.get('readonly', False)
            description = reg.get('description', '')
            self.global_registers[name] = dict(name=name, address=address, offset=offset, bitlength=bitlength, addresses=addresses, littleendian=littleendian, register_littleendian=register_littleendian, value=value, readonly=readonly, description=description)
        for name, reg in fe_type['pixel_registers'].iteritems():
            pxstrobe = reg.get('pxstrobe')
            bitlength = reg.get('bitlength')
            if bitlength > 8:
                raise Exception('Pixel register %s: up to 8 bits supported' % (name,))  # numpy array dtype is uint8
            littleendian = reg.get('littleendian', False)
            if not 0 <= reg.get('value', 0) < 2 ** bitlength:
                raise ValueError("Global register %s: value exceeds limits" % (name,))
            value = np.full((80, 336), reg.get('value', 0), dtype=np.uint8)
            description = reg.get('description', '')
            self.pixel_registers[name] = dict(name=name, pxstrobe=pxstrobe, bitlength=bitlength, littleendian=littleendian, value=value, description=description)
        for name, command in fe_type['commands'].iteritems():
            bitlength = command.get('bitlength')
            description = command.get('description', '')
            if 'bitstream' in command:
                bitstream = command.get('bitstream')
                self.commands[name] = dict(name=name, bitstream=bitstream, bitlength=bitlength, description=description)
            else:
                self.commands[name] = dict(name=name, bitlength=bitlength, description=description)
        self.calibration_parameters = fe_type['calibration_parameters'].copy()

    def is_chip_flavor(self, chip_flavor):
        if chip_flavor in flavors:
            if chip_flavor == self.flavor:
                return True
            else:
                return False
        else:
            raise ValueError('Unknown FEI4 flavor: %s' % chip_flavor)

    @property
    def chip_flavor(self):
        return self.flavor

    @property
    def fei4a(self):
        return True if self.flavor == 'fei4a' else False

    @property
    def fei4b(self):
        return True if self.flavor == 'fei4b' else False

    def load_configuration(self, configuration_file):
        '''Loading configuration

        Parameters
        ----------
        configuration_file : string
            Path to the configuration file (text or HDF5 file).
        '''
        if os.path.isfile(configuration_file):
            if not isinstance(configuration_file, tb.file.File) and os.path.splitext(configuration_file)[1].strip().lower() != ".h5":
                self._load_configuration_from_text_file(configuration_file)
            else:
                self._load_configuration_from_hdf5(configuration_file)
        else:
            raise ValueError('Cannot find configuration file specified: %s' % configuration_file)

    def save_configuration(self, configuration_file):
        '''Saving configuration

        Parameters
        ----------
        configuration_file : string
            Filename of the configuration file.
        '''
        if not isinstance(configuration_file, tb.file.File) and os.path.splitext(configuration_file)[1].strip().lower() != ".h5":
            return self._save_configuration_to_text_file(configuration_file)
        else:
            return self._save_configuration_to_hdf5(configuration_file)

    def _load_configuration_from_text_file(self, configuration_file):
        '''Loading configuration from text files

        Parameters
        ----------
        configuration_file : string
            Full path (directory and filename) of the configuration file. If name is not given, reload configuration from file.
        '''
        logging.info("Loading configuration: %s" % configuration_file)
        self.configuration_file = configuration_file
        with open(configuration_file, 'r') as f:
            f.seek(0)
            config_dict = {}
            for line in f.readlines():
                line = line.partition('#')[0].strip()
                if not line:
                    continue
                parts = re.split(r'\s*[=]\s*|\s+', line)
                if parts[0] in config_dict:
                    logging.warning('Item %s in configuration file exists more than once' % parts[0])
                try:
                    config_dict[parts[0]] = ast.literal_eval(parts[1])
                except SyntaxError:
                    config_dict[parts[0]] = parts[1].strip()
                except ValueError:
                    config_dict[parts[0]] = parts[1].strip()

        if 'Flavor' in config_dict:
            flavor = config_dict.pop('Flavor')
            if self.flavor:
                pass
            else:
                self.init_fe_type(flavor)
        else:
            if self.flavor:
                pass
            else:
                raise ValueError('Flavor not specified')
        if 'Chip_ID' in config_dict:
            chip_id = config_dict.pop('Chip_ID')
            if self.chip_address:
                pass
            else:
                self.broadcast = True if chip_id & 0x8 else False
                self.set_chip_address(chip_id & 0x7)
        elif 'Chip_Address' in config_dict:
            chip_address = config_dict.pop('Chip_Address')
            if self.chip_address:
                pass
            else:
                self.set_chip_address(chip_address)
        else:
            if self.chip_id_initialized:
                pass
            else:
                raise ValueError('Chip address not specified')
        global_registers_configured = []
        pixel_registers_configured = []
        for key in config_dict.keys():
            value = config_dict.pop(key)
            if key in self.global_registers:
                self.set_global_register_value(key, value)
                global_registers_configured.append(key)
            elif key in self.pixel_registers:
                self.set_pixel_register_value(key, value)
                pixel_registers_configured.append(key)
            elif key in self.calibration_parameters:
                self.calibration_parameters[key] = value
            else:
                self.miscellaneous[key] = value

        global_registers = self.get_global_register_attributes('name', readonly=False)
        pixel_registers = self.pixel_registers.keys()
        global_registers_not_configured = set(global_registers).difference(global_registers_configured)
        pixel_registers_not_configured = set(pixel_registers).difference(pixel_registers_configured)
        if global_registers_not_configured:
            logging.warning("Following global register(s) not configured: {}".format(', '.join('\'' + reg + '\'' for reg in global_registers_not_configured)))
        if pixel_registers_not_configured:
            logging.warning("Following pixel register(s) not configured: {}".format(', '.join('\'' + reg + '\'' for reg in pixel_registers_not_configured)))
        if self.miscellaneous:
            logging.warning("Found following unknown parameter(s): {}".format(', '.join('\'' + parameter + '\'' for parameter in self.miscellaneous.iterkeys())))

    def _save_configuration_to_text_file(self, configuration_file):
        '''Saving configuration to text files

        Parameters
        ----------
        configuration_file : string
            Filename of the configuration file.
        '''
        configuration_path, filename = os.path.split(configuration_file)
        if os.path.split(configuration_path)[1] == 'configs':
            configuration_path = os.path.split(configuration_path)[0]
        filename = os.path.splitext(filename)[0].strip()
        self.configuration_file = os.path.join(os.path.join(configuration_path, 'configs'), filename + ".cfg")
        if os.path.isfile(self.configuration_file):
            logging.warning("Overwriting configuration: %s" % self.configuration_file)
        else:
            logging.info("Saving configuration: %s" % self.configuration_file)
        pixel_reg_dict = {}
        for path in ["tdacs", "fdacs", "masks", "configs"]:
            configuration_file_path = os.path.join(configuration_path, path)
            if not os.path.exists(configuration_file_path):
                os.makedirs(configuration_file_path)
            if path == "tdacs":
                dac = self.get_pixel_register_objects(name="TDAC")[0]
                dac_config_path = os.path.join(configuration_file_path, "_".join([dac['name'].lower(), filename]) + ".dat")
                write_pixel_dac_config(dac_config_path, dac['value'])
                pixel_reg_dict[dac['name']] = os.path.relpath(dac_config_path, os.path.dirname(self.configuration_file))
            elif path == "fdacs":
                dac = self.get_pixel_register_objects(name="FDAC")[0]
                dac_config_path = os.path.join(configuration_file_path, "_".join([dac['name'].lower(), filename]) + ".dat")
                write_pixel_dac_config(dac_config_path, dac['value'])
                pixel_reg_dict[dac['name']] = os.path.relpath(dac_config_path, os.path.dirname(self.configuration_file))
            elif path == "masks":
                masks = self.get_pixel_register_objects(bitlength=1)
                for mask in masks:
                    dac_config_path = os.path.join(configuration_file_path, "_".join([mask['name'].lower(), filename]) + ".dat")
                    write_pixel_mask_config(dac_config_path, mask['value'])
                    pixel_reg_dict[mask['name']] = os.path.relpath(dac_config_path, os.path.dirname(self.configuration_file))
            elif path == "configs":
                with open(self.configuration_file, 'w') as f:
                    lines = []
                    lines.append("# FEI4 Flavor\n")
                    lines.append('%s %s\n' % ('Flavor', self.flavor))
                    lines.append("\n# FEI4 Chip ID\n")
                    lines.append('%s %d\n' % ('Chip_ID', self.chip_id))
                    lines.append("\n# FEI4 Global Registers\n")
                    global_regs = self.get_global_register_objects(readonly=False)
                    for global_reg in sorted(global_regs, key=itemgetter('name')):
                        lines.append('%s %d\n' % (global_reg['name'], global_reg['value']))
                    lines.append("\n# FEI4 Pixel Registers\n")
                    for key in sorted(pixel_reg_dict):
                        lines.append('%s %s\n' % (key, pixel_reg_dict[key]))
                    if self.miscellaneous:
                        lines.append("\n# Miscellaneous\n")
                        for key, value in self.miscellaneous.iteritems():
                            lines.append('%s %s\n' % (key, value))
                    f.writelines(lines)

    def _load_configuration_from_hdf5(self, configuration_file, node=''):
        '''Loading configuration from HDF5 file

        Parameters
        ----------
        configuration_file : string, file
            Filename of the HDF5 configuration file or file object.
        node : string
            Additional identifier (subgroup). Useful when more than one configuration is stored inside a HDF5 file.
        '''
        def load_conf():
            logging.info("Loading configuration: %s" % h5_file.filename)
            self.configuration_file = h5_file.filename
            if node:
                configuration_group = h5_file.root.configuration.node
            else:
                configuration_group = h5_file.root.configuration

            # miscellaneous
            for row in configuration_group.miscellaneous:
                name = row['name']
                try:
                    value = ast.literal_eval(row['value'])
                except ValueError:
                    value = row['value']
                if name == 'Flavor':
                    if self.flavor:
                        pass
                    else:
                        self.init_fe_type(value)
                elif name == 'Chip_ID':
                    if self.chip_address:
                        pass
                    else:
                        self.broadcast = True if value & 0x8 else False
                        self.set_chip_address(value & 0x7)
                elif name == 'Chip_Address':
                    if self.chip_address:
                        pass
                    else:
                        self.set_chip_address(value)
                else:
                    self.miscellaneous[name] = value

            if self.flavor:
                pass
            else:
                raise ValueError('Flavor not specified')

            if self.chip_id_initialized:
                pass
            else:
                raise ValueError('Chip address not specified')

            # calibration parameters
            for row in configuration_group.calibration_parameters:
                name = row['name']
                value = row['value']
                self.calibration_parameters[name] = ast.literal_eval(value)

            # global
            for row in configuration_group.global_register:
                name = row['name']
                value = row['value']
                self.set_global_register_value(name, ast.literal_eval(value))

            # pixels
            for pixel_reg in h5_file.iter_nodes(configuration_group, 'CArray'):  # ['Enable', 'TDAC', 'C_High', 'C_Low', 'Imon', 'FDAC', 'EnableDigInj']:
                if pixel_reg.name in self.pixel_registers:
                    self.set_pixel_register_value(pixel_reg.name, np.asarray(pixel_reg).T)  # np.asarray(h5_file.get_node(configuration_group, name=pixel_reg)).T

        if isinstance(configuration_file, tb.file.File):
            h5_file = configuration_file
            load_conf()
        else:
            with tb.open_file(configuration_file, mode="r", title='') as h5_file:
                load_conf()

    def _save_configuration_to_hdf5(self, configuration_file, name=''):
        '''Saving configuration to HDF5 file

        Parameters
        ----------
        configuration_file : string, file
            Filename of the HDF5 configuration file or file object.
        name : string
            Additional identifier (subgroup). Useful when storing more than one configuration inside a HDF5 file.
        '''
        def save_conf():
            logging.info("Saving configuration: %s" % h5_file.filename)
            self.configuration_file = h5_file.filename
            try:
                configuration_group = h5_file.create_group(h5_file.root, "configuration")
            except tb.NodeError:
                configuration_group = h5_file.root.configuration
            if name:
                try:
                    configuration_group = h5_file.create_group(configuration_group, name)
                except tb.NodeError:
                    configuration_group = h5_file.root.configuration.name

            # calibration_parameters
            try:
                h5_file.remove_node(configuration_group, name='calibration_parameters')
            except tb.NodeError:
                pass
            calibration_data_table = h5_file.create_table(configuration_group, name='calibration_parameters', description=NameValue, title='calibration_parameters')
            calibration_data_row = calibration_data_table.row
            for key, value in self.calibration_parameters.iteritems():
                calibration_data_row['name'] = key
                calibration_data_row['value'] = value
                calibration_data_row.append()
            calibration_data_table.flush()

            # miscellaneous
            try:
                h5_file.remove_node(configuration_group, name='miscellaneous')
            except tb.NodeError:
                pass
            miscellaneous_data_table = h5_file.create_table(configuration_group, name='miscellaneous', description=NameValue, title='miscellaneous')
            miscellaneous_data_row = miscellaneous_data_table.row
            miscellaneous_data_row['name'] = 'Flavor'
            miscellaneous_data_row['value'] = self.flavor
            miscellaneous_data_row.append()
            miscellaneous_data_row['name'] = 'Chip_ID'
            miscellaneous_data_row['value'] = self.chip_id
            miscellaneous_data_row.append()
            for key, value in self.miscellaneous.iteritems():
                miscellaneous_data_row['name'] = key
                miscellaneous_data_row['value'] = value
                miscellaneous_data_row.append()
            miscellaneous_data_table.flush()

            # global
            try:
                h5_file.remove_node(configuration_group, name='global_register')
            except tb.NodeError:
                pass
            global_data_table = h5_file.create_table(configuration_group, name='global_register', description=NameValue, title='global_register')
            global_data_table_row = global_data_table.row
            global_regs = self.get_global_register_objects(readonly=False)
            for global_reg in sorted(global_regs, key=itemgetter('name')):
                global_data_table_row['name'] = global_reg['name']
                global_data_table_row['value'] = global_reg['value']  # TODO: some function that converts to bin, hex
                global_data_table_row.append()
            global_data_table.flush()

            # pixel
            for pixel_reg in self.pixel_registers.itervalues():
                try:
                    h5_file.remove_node(configuration_group, name=pixel_reg['name'])
                except tb.NodeError:
                    pass
                data = pixel_reg['value'].T
                atom = tb.Atom.from_dtype(data.dtype)
                ds = h5_file.createCArray(configuration_group, name=pixel_reg['name'], atom=atom, shape=data.shape, title=pixel_reg['name'])
                ds[:] = data

        if isinstance(configuration_file, tb.file.File):
            h5_file = configuration_file
            save_conf()
        else:
            with tb.open_file(configuration_file, mode="a", title='') as h5_file:
                save_conf()

    def parse_parameters(self, parameters):
        with open(self.configuration_file, 'r') as f:
            for line in f.readlines():
                key_value = re.split(r'\s*[=]\s*|\s+', line)
                if key_value[0] in parameters:
                    try:
                        parameters[key_value[0]] = ast.literal_eval(key_value[1].strip())
                    except SyntaxError:  # for comma separated values, e.g. lists
                        parameters[key_value[0]] = ast.literal_eval(line[len(key_value[0]):].strip())
                    except ValueError:
                        parameters[key_value[0]] = key_value[1].strip().lower()

    def write_parameters(self, parameters, title=None):
        with open(self.configuration_file, 'a') as f:
            lines = []
            if title:
                lines.append("# %s\n" % title)
            for key, value in parameters.iteritems():
                lines.append('%s %s\n' % (key, str(value)))
            lines.append("\n")
            f.writelines(lines)

    '''
    TODO:
    for the following functions use
    filter(function, iterable).

    Make new generic function that uses filter.

    Use next(iterator[, default]).
    '''

    def set_global_register_value(self, name, value):
        if self.global_registers[name]['readonly']:
            raise ValueError('Global register %s: register is read-only' % name)
        value = long(str(value), 0)  # value is decimal string or number or BitVector
        if not 0 <= self.global_registers[name]['value'] < 2 ** self.global_registers[name]['bitlength']:
            raise ValueError('Global register %s: value exceeds limits' % name)
        self.global_registers[name]['value'] = value

    def get_global_register_value(self, name):
        return self.global_registers[name]['value']

    def set_pixel_register_value(self, name, value):
        try:  # value is decimal string or number or array
            self.pixel_registers[name]['value'][:, :] = value
        except ValueError:  # value is path to pixel config
            if self.pixel_registers[name]['bitlength'] == 1:  # pixel mask
                if value[0] == "~" or value[0] == "!":
                    reg_value = parse_pixel_mask_config(os.path.join(os.path.dirname(self.configuration_file), value[1:]))
                    inverted_mask = np.ones(shape=(80, 336), dtype=np.dtype('>u1'))
                    inverted_mask[reg_value >= 1] = 0
                    self.pixel_registers[name]['value'][:, :] = inverted_mask
                else:
                    self.pixel_registers[name]['value'][:, :] = parse_pixel_mask_config(os.path.join(os.path.dirname(self.configuration_file), value))
            else:  # pixel dac
                self.pixel_registers[name]['value'][:, :] = parse_pixel_dac_config(os.path.join(os.path.dirname(self.configuration_file), value))
        if (self.pixel_registers[name]['value'] >= 2 ** self.pixel_registers[name]['bitlength']).any() or (self.pixel_registers[name]['value'] < 0).any():
            raise ValueError("Pixel register %s: value exceeds limits" % name)

    def get_pixel_register_value(self, name):
        return self.pixel_registers[name]['value'].copy()

    def get_commands(self, command_name, **kwargs):
        """get fe_command from command name and keyword arguments

        wrapper for build_commands()
        implements FEI4 specific behavior

        """
        chip_id = kwargs.pop("ChipID", self.chip_id_bitarray)
        commands = []
        if command_name == "zeros":
            if "length" in kwargs:
                bv = bitarray(kwargs["length"], endian='little')  # all bits to zero
            elif "mask_steps" in kwargs:
{'fraction_non_alphanumeric': 0.105782865838896, 'fraction_numerical': 0.013627088161034829, 'mean_word_length': 2.045930998156439, 'pattern_counts': {'":': 16, '<': 21, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '635627', 'n_tokens_mistral': 16678, 'n_tokens_neox': 15606, 'n_words': 3710}