<commit_before>import logging
import os
import pickle
import time
import traceback
from typing import List, Any

from ..devices.device import Device, DeviceBackend_ModbusTCP, DeviceBackend_TCP, DeviceError
from ..devices.motor import Motor
from ..services import Accounting, ExposureAnalyzer, FileSequence, Interpreter, SampleStore, Service, TelemetryManager, \
    WebStateFileWriter
from ..utils.callback import Callbacks, SignalFlags
from ..utils.telemetry import TelemetryInfo
from ..utils.timeout import TimeOut

logger = logging.getLogger(__name__)
logger.setLevel(logging.INFO)


class DummyTm(object):
    ru_utime = None
    ru_stime = None
    ru_maxrss = None
    ru_minflt = None
    ru_majflt = None
    ru_inblock = None
    ru_oublock = None
    ru_nvcsw = None
    ru_nivcsw = None


class InstrumentError(Exception):
    pass


class Instrument(Callbacks):
    configdir = 'config'
    telemetry_timeout = 0.9
    statusfile_timeout = 30
    memlog_timeout = 60  # write memory usage log
    memlog_file = 'memory.log'

    __signals__ = {
        # emitted when all devices have been initialized.
        'devices-ready': (SignalFlags.RUN_FIRST, None, ()),
        # emitted when telemetry data is obtained from a component. The first
        # argument is the name of the component, the second is the type of the
        # component (service or device), and the third one is the telemetry
        # dictionary.
        'telemetry': (SignalFlags.RUN_FIRST, None, (object, str, object)),
        # emitted when the instrument is finalized: all devices disconnected,
        # all services stopped.
        'shutdown': (SignalFlags.RUN_FIRST, None, ()),
        # emitted when a device is connected successfully. The argument is the device.
        'device-connected': (SignalFlags.RUN_FIRST, None, (object,)),
        # emitted when a device is disconnected. The first argument is the device,
        # the second one is a bool: if the disconnection was expected or not.
        'device-disconnected': (SignalFlags.RUN_FIRST, None, (object, bool)),
        # Emitted whenever the configuration changes. Note that the emission of this
        # signal is not automatic: it must be ensured by the user by calling
        # Instrument.emit_config_change_signal(). The signal is emitted automatically
        # upon loading and saving the state.
        'config-changed': (SignalFlags.RUN_FIRST, None, ()),
    }

    def __init__(self, online: bool):
        Callbacks.__init__(self)
        self._online = online
        self.shutdown_requested = False
        self.devices = {}
        self.pseudo_devices = {}
        self.services = {}
        self.motors = {}
        self.configfile = os.path.join(self.configdir, 'cct.pickle')
        self._initialize_config()
        self._signalconnections = {}
        self._waiting_for_ready = []
        self._telemetries = {}
        self._telemetry_timeout = None
        self._service_connections = {}
        self.starttime = None
        self.load_state()
        self.create_services()

    def emit_config_change_signal(self):
        self.emit('config-changed')

    @property
    def online(self) -> bool:
        return self._online

    def is_busy(self) -> bool:
        return any([s.is_busy() for s in self.services.values()])

    def start(self):
        """Start operation"""
        self.starttime = time.monotonic()
        self._telemetry_timeout = TimeOut(self.telemetry_timeout * 1000,
                                          self.on_telemetry_timeout)
        for s in self.services:
            self.services[s].start()
        logger.info('Started all services.')

    # noinspection PyDictCreation
    def _initialize_config(self):
        """Create a sane configuration in `self.config` from scratch."""
        self.config = {
            'path': {
                'directories': {
                    'log': 'log',
                    'images': 'images',
                    'param': 'param',
                    'config': 'config',
                    'mask': 'mask',
                    'nexus': 'nexus',
                    'eval1d': 'eval1d',
                    'eval2d': 'eval2d',
                    'param_override': 'param_override',
                    'scan': 'scan',
                    'images_detector': ['/disk2/images', '/home/det/p2_det/images'],
                    'status': 'status',
                    'scripts': 'scripts',
                },
                'fsndigits': 5,
                'prefixes': {
                    'crd': 'crd',
                    'scn': 'scn',
                    'tra': 'tra',
                    'tst': 'tst'
                },
                'varlogfile': 'varlog.log'
            },
            'geometry': {
                'dist_sample_det': 1000.,
                'dist_sample_det.err': 0.,
                'dist_source_ph1': 100.,
                'dist_ph1_ph2': 100.,
                'dist_ph2_ph3': 1.,
                'dist_ph3_sample': 2.,
                'dist_det_beamstop': 1.,
                'pinhole_1': 1000.,
                'pinhole_2': 300.,
                'pinhole_3': 750.,
                'description': 'Generic geometry, please correct values',
                'beamstop': 4.,
                'wavelength': 0.15418,
                'wavelength.err': 0.15418 * 0.03,
                'beamposx': 330.,
                'beamposy': 257.,
                'beamposx.err': 0.,
                'beamposy.err': 0.,
                'pixelsize': 0.172,
                'mask': 'mask.mat'
            },
            'connections': {
                'xray_source': {
                    'host': 'genix.credo',
                    'port': 502,
                    'timeout': 1,
                    'name': 'genix',
                    'classname': 'GeniX'
                },
                'detector': {
                    'host': 'pilatus300k.credo',
                    'port': 41234,
                    'timeout': 0.01,
                    'poll_timeout': 0.01,
                    'name': 'pilatus',
                    'classname': 'Pilatus'
                },
                'vacuum': {
                    'host': 'devices.credo',
                    'port': 2006,
                    'timeout': 0.1,
                    'poll_timeout': 0.1,
                    'name': 'tpg201',
                    'classname': 'TPG201'
                },
                'temperature': {
                    'host': 'devices.credo',
                    'port': 2001,<commit_msg>uncertainty of pixel size in the config<commit_after>import logging
import os
import pickle
import time
import traceback
from typing import List, Any

from ..devices.device import Device, DeviceBackend_ModbusTCP, DeviceBackend_TCP, DeviceError
from ..devices.motor import Motor
from ..services import Accounting, ExposureAnalyzer, FileSequence, Interpreter, SampleStore, Service, TelemetryManager, \
    WebStateFileWriter
from ..utils.callback import Callbacks, SignalFlags
from ..utils.telemetry import TelemetryInfo
from ..utils.timeout import TimeOut

logger = logging.getLogger(__name__)
logger.setLevel(logging.INFO)


class DummyTm(object):
    ru_utime = None
    ru_stime = None
    ru_maxrss = None
    ru_minflt = None
    ru_majflt = None
    ru_inblock = None
    ru_oublock = None
    ru_nvcsw = None
    ru_nivcsw = None


class InstrumentError(Exception):
    pass


class Instrument(Callbacks):
    configdir = 'config'
    telemetry_timeout = 0.9
    statusfile_timeout = 30
    memlog_timeout = 60  # write memory usage log
    memlog_file = 'memory.log'

    __signals__ = {
        # emitted when all devices have been initialized.
        'devices-ready': (SignalFlags.RUN_FIRST, None, ()),
        # emitted when telemetry data is obtained from a component. The first
        # argument is the name of the component, the second is the type of the
        # component (service or device), and the third one is the telemetry
        # dictionary.
        'telemetry': (SignalFlags.RUN_FIRST, None, (object, str, object)),
        # emitted when the instrument is finalized: all devices disconnected,
        # all services stopped.
        'shutdown': (SignalFlags.RUN_FIRST, None, ()),
        # emitted when a device is connected successfully. The argument is the device.
        'device-connected': (SignalFlags.RUN_FIRST, None, (object,)),
        # emitted when a device is disconnected. The first argument is the device,
        # the second one is a bool: if the disconnection was expected or not.
        'device-disconnected': (SignalFlags.RUN_FIRST, None, (object, bool)),
        # Emitted whenever the configuration changes. Note that the emission of this
        # signal is not automatic: it must be ensured by the user by calling
        # Instrument.emit_config_change_signal(). The signal is emitted automatically
        # upon loading and saving the state.
        'config-changed': (SignalFlags.RUN_FIRST, None, ()),
    }

    def __init__(self, online: bool):
        Callbacks.__init__(self)
        self._online = online
        self.shutdown_requested = False
        self.devices = {}
        self.pseudo_devices = {}
        self.services = {}
        self.motors = {}
        self.configfile = os.path.join(self.configdir, 'cct.pickle')
        self._initialize_config()
        self._signalconnections = {}
        self._waiting_for_ready = []
        self._telemetries = {}
        self._telemetry_timeout = None
        self._service_connections = {}
        self.starttime = None
        self.load_state()
        self.create_services()

    def emit_config_change_signal(self):
        self.emit('config-changed')

    @property
    def online(self) -> bool:
        return self._online

    def is_busy(self) -> bool:
        return any([s.is_busy() for s in self.services.values()])

    def start(self):
        """Start operation"""
        self.starttime = time.monotonic()
        self._telemetry_timeout = TimeOut(self.telemetry_timeout * 1000,
                                          self.on_telemetry_timeout)
        for s in self.services:
            self.services[s].start()
        logger.info('Started all services.')

    # noinspection PyDictCreation
    def _initialize_config(self):
        """Create a sane configuration in `self.config` from scratch."""
        self.config = {
            'path': {
                'directories': {
                    'log': 'log',
                    'images': 'images',
                    'param': 'param',
                    'config': 'config',
                    'mask': 'mask',
                    'nexus': 'nexus',
                    'eval1d': 'eval1d',
                    'eval2d': 'eval2d',
                    'param_override': 'param_override',
                    'scan': 'scan',
                    'images_detector': ['/disk2/images', '/home/det/p2_det/images'],
                    'status': 'status',
                    'scripts': 'scripts',
                },
                'fsndigits': 5,
                'prefixes': {
                    'crd': 'crd',
                    'scn': 'scn',
                    'tra': 'tra',
                    'tst': 'tst'
                },
                'varlogfile': 'varlog.log'
            },
            'geometry': {
                'dist_sample_det': 1000.,
                'dist_sample_det.err': 0.,
                'dist_source_ph1': 100.,
                'dist_ph1_ph2': 100.,
                'dist_ph2_ph3': 1.,
                'dist_ph3_sample': 2.,
                'dist_det_beamstop': 1.,
                'pinhole_1': 1000.,
                'pinhole_2': 300.,
                'pinhole_3': 750.,
                'description': 'Generic geometry, please correct values',
                'beamstop': 4.,
                'wavelength': 0.15418,
                'wavelength.err': 0.15418 * 0.03,
                'beamposx': 330.,
                'beamposy': 257.,
                'beamposx.err': 0.,
                'beamposy.err': 0.,
                'pixelsize': 0.1721,
                'pixelsize.err': 0.0002,
                'mask': 'mask.mat'
            },
            'connections': {
                'xray_source': {
                    'host': 'genix.credo',
                    'port': 502,
                    'timeout': 1,
                    'name': 'genix',
                    'classname': 'GeniX'
                },
                'detector': {
                    'host': 'pilatus300k.credo',
                    'port': 41234,
                    'timeout': 0.01,
                    'poll_timeout': 0.01,
                    'name': 'pilatus',
                    'classname': 'Pilatus'
                },
                'vacuum': {
                    'host': 'devices.credo',
                    'port': 2006,
                    'timeout': 0.1,
                    'poll_timeout': 0.1,
                    'name': 'tpg201',
                    'classname': 'TPG201'
                },
                'temperature': {
                    'host': 'devices.credo',
                    'port': 2001,
{'fraction_non_alphanumeric': 0.11011270179713677, 'fraction_numerical': 0.017819067925677733, 'mean_word_length': 1.5928923988153998, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4019604', 'n_tokens_mistral': 3868, 'n_tokens_neox': 3492, 'n_words': 913}