<commit_before>"""A script that runs a fast threshold scan for different parameter (e.g. GDAC, TDACVbp) to get a threshold calibration. 
To save time the PlsrDAC start position is the start position determined from the previous threshold scan. So the
scan parameter values should be chosen in a ways that the threshold increases for each step.
After the data taking the data is analyzed and the calibration is written into a h5 file.
"""

import tables as tb
import numpy as np
import logging
import ast

from matplotlib.backends.backend_pdf import PdfPages

from pybar.run_manager import RunManager
from pybar.scans.scan_threshold_fast import FastThresholdScan
from pybar.analysis import analysis_utils
from pybar.analysis.RawDataConverter import data_struct
from pybar.analysis.plotting.plotting import plotThreeWay, plot_scurves, plot_scatter
from pybar.analysis.analyze_raw_data import AnalyzeRawData


class ThresholdCalibration(FastThresholdScan):

    ''' Threshold calibration scan
    '''
    _default_run_conf = FastThresholdScan._default_run_conf.copy()
    _default_run_conf['scan_parameters'] = [('PlsrDAC', (0, None)), ('GDAC', np.unique(np.logspace(1.7, 4.0, 10).astype(np.int)).tolist())]
    _default_run_conf.update({
        "ignore_columns": (1, 78, 79, 80),
        "ignore_parameter_values": None,  # do not use data for these parameter values for the calibration
        "create_plots": True,
        "send_data": 'tcp://127.0.0.1:5678',  # send data to socket (e.g. for Online Monitor)
    })

    def scan(self):
        logging.info('Taking threshold data at following ' + self.scan_parameters._fields[1] + ' values: %s', str(self.scan_parameters[1]))

        for index, parameter_value in enumerate(self.scan_parameters[1]):
            if self.scan_parameters._fields[1] == 'GDAC':
                self.register_utils.set_gdac(parameter_value)
            else:
                self.register.set_global_register_value(self.scan_parameters._fields[1], parameter_value)

            if index == 0:
                actual_scan_parameters = {'PlsrDAC': self.scan_parameters.PlsrDAC, self.scan_parameters._fields[1]: parameter_value}
            else:
                self.minimum_data_points = self.data_points
                actual_scan_parameters = {'PlsrDAC': (self.scan_parameter_start, None), self.scan_parameters._fields[1]: parameter_value}
            self.set_scan_parameters(**actual_scan_parameters)
            super(ThresholdCalibration, self).scan()
        logging.info("Finished!")

    def handle_data(self, data):
        self.raw_data_file.append_item(data, scan_parameters=self.scan_parameters._asdict(), new_file=[self.scan_parameters._fields[1]], flush=False)

    def analyze(self):
        def analyze_raw_data_file(file_name):
            with AnalyzeRawData(raw_data_file=file_name, create_pdf=False) as analyze_raw_data:
                analyze_raw_data.create_tot_hist = False
                analyze_raw_data.create_fitted_threshold_hists = True
                analyze_raw_data.create_threshold_mask = True
                analyze_raw_data.interpreter.set_warning_output(True)  # so far the data structure in a threshold scan was always bad, too many warnings given
                analyze_raw_data.interpret_word_table()

        def store_calibration_data_as_table(out_file_h5, mean_threshold_calibration, mean_threshold_rms_calibration, threshold_calibration, parameter_values):
            logging.info("Storing calibration data in a table...")
            filter_table = tb.Filters(complib='blosc', complevel=5, fletcher32=False)
            mean_threshold_calib_table = out_file_h5.createTable(out_file_h5.root, name='MeanThresholdCalibration', description=data_struct.MeanThresholdCalibrationTable, title='mean_threshold_calibration', filters=filter_table)<commit_msg>MAINT: avoid abort due to RX, issued #6<commit_after>"""A script that runs a fast threshold scan for different parameter (e.g. GDAC, TDACVbp) to get a threshold calibration. 
To save time the PlsrDAC start position is the start position determined from the previous threshold scan. So the
scan parameter values should be chosen in a ways that the threshold increases for each step.
After the data taking the data is analyzed and the calibration is written into a h5 file.
"""

import tables as tb
import numpy as np
import logging
import ast

from matplotlib.backends.backend_pdf import PdfPages

from pybar.run_manager import RunManager
from pybar.scans.scan_threshold_fast import FastThresholdScan
from pybar.analysis import analysis_utils
from pybar.analysis.RawDataConverter import data_struct
from pybar.analysis.plotting.plotting import plotThreeWay, plot_scurves, plot_scatter
from pybar.analysis.analyze_raw_data import AnalyzeRawData


class ThresholdCalibration(FastThresholdScan):

    ''' Threshold calibration scan
    '''
    _default_run_conf = FastThresholdScan._default_run_conf.copy()
    _default_run_conf['scan_parameters'] = [('PlsrDAC', (0, None)), ('GDAC', np.unique(np.logspace(1.7, 4.0, 10).astype(np.int)).tolist())]
    _default_run_conf.update({
        "ignore_columns": (1, 78, 79, 80),
        'reset_rx_on_error': True,  # long scans have a high propability for ESD related data transmission errors; recover and continue here
        "create_plots": True,
        "send_data": 'tcp://127.0.0.1:5678',  # send data to socket (e.g. for Online Monitor)
    })

    def scan(self):
        logging.info('Taking threshold data at following ' + self.scan_parameters._fields[1] + ' values: %s', str(self.scan_parameters[1]))

        for index, parameter_value in enumerate(self.scan_parameters[1]):
            if self.scan_parameters._fields[1] == 'GDAC':
                self.register_utils.set_gdac(parameter_value)
            else:
                self.register.set_global_register_value(self.scan_parameters._fields[1], parameter_value)

            if index == 0:
                actual_scan_parameters = {'PlsrDAC': self.scan_parameters.PlsrDAC, self.scan_parameters._fields[1]: parameter_value}
            else:
                self.minimum_data_points = self.data_points
                actual_scan_parameters = {'PlsrDAC': (self.scan_parameter_start, None), self.scan_parameters._fields[1]: parameter_value}
            self.set_scan_parameters(**actual_scan_parameters)
            super(ThresholdCalibration, self).scan()
        logging.info("Finished!")

    def handle_data(self, data):
        self.raw_data_file.append_item(data, scan_parameters=self.scan_parameters._asdict(), new_file=[self.scan_parameters._fields[1]], flush=False)

    def analyze(self):
        def analyze_raw_data_file(file_name):
            with AnalyzeRawData(raw_data_file=file_name, create_pdf=False) as analyze_raw_data:
                analyze_raw_data.create_tot_hist = False
                analyze_raw_data.create_fitted_threshold_hists = True
                analyze_raw_data.create_threshold_mask = True
                analyze_raw_data.interpreter.set_warning_output(False)  # RX errors would fill the console
                analyze_raw_data.interpret_word_table()

        def store_calibration_data_as_table(out_file_h5, mean_threshold_calibration, mean_threshold_rms_calibration, threshold_calibration, parameter_values):
            logging.info("Storing calibration data in a table...")
            filter_table = tb.Filters(complib='blosc', complevel=5, fletcher32=False)
            mean_threshold_calib_table = out_file_h5.createTable(out_file_h5.root, name='MeanThresholdCalibration', description=data_struct.MeanThresholdCalibrationTable, title='mean_threshold_calibration', filters=filter_table)
{'fraction_non_alphanumeric': 0.08617992177314211, 'fraction_numerical': 0.010560625814863103, 'mean_word_length': 3.907869481765835, 'pattern_counts': {'":': 7, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 2, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '108757', 'n_tokens_mistral': 2271, 'n_tokens_neox': 2169, 'n_words': 531}