<commit_before># -*- coding: utf-8 -*-

from pyqtgraph.flowchart.library.common import *
import functions
import numpy as np
import scipy
#from pyqtgraph import graphicsItems
import pyqtgraph as pg
import metaarray
#import pyqtgraph.CheckTable as CheckTable
from collections import OrderedDict

class EventFitter(CtrlNode):
    """Takes a waveform and event list as input, returns extra information about each event.
    Optionally performs an exponential reconvolution before measuring each event.
    Plots fits of reconstructed events if the plot output is connected."""
    nodeName = "EventFitter"
    uiTemplate = [
        ('multiFit', 'check', {'value': False}),
        #('parallel', 'check', {'value': False}),
        #('nProcesses', 'spin', {'value': 1, 'min': 1, 'int': True}),
        ('plotFits', 'check', {'value': True}),
        ('plotGuess', 'check', {'value': False}),
        ('plotEvents', 'check', {'value': False}),
    ]
    
    
    def __init__(self, name):
        CtrlNode.__init__(self, name, terminals={
            'waveform': {'io': 'in'},
            'events': {'io': 'in'},
            'output': {'io': 'out'},
            'plot':  {'io': 'out'}
        })
        self.plotItems = []
        self.selectedFit = None
        self.deletedFits = []
        self.pool = None  ## multiprocessing pool
        self.poolSize = 0
        #self.ctrls['parallel'].toggled.connect(self.setupPool)
        #self.ctrls['nProcesses'].valueChanged.connect(self.setupPool)
    
    #def setupPool(self):
        #import multiprocessing as mp
        #if self.ctrls['parallel'].isChecked():
            #nProc = self.ctrls['nProcesses'].value()
            #if self.pool is not None and self.poolSize != nProc:
                #self.pool.terminate()
                #self.pool = None
            #if self.pool is None:
                #self.pool = mp.Pool(processes=nProc)
                #self.poolSize = nProc
        #else:
            #if self.pool is not None:
                #self.pool.terminate()
                #self.pool = None
    
    def process(self, waveform, events, display=True):
        self.deletedFits = []
        for item in self.plotItems:
            try:
                item.sigClicked.disconnect(self.fitClicked)
            except:
                pass
        self.plotItems = []
        
        tau = waveform.infoCopy(-1).get('expDeconvolveTau', None)
        dt = waveform.xvals(0)[1] - waveform.xvals(0)[0]
        opts = {
            'dt': dt, 'tau': tau, 'multiFit': self.ctrls['multiFit'].isChecked(),
            'waveform': waveform.view(np.ndarray),
            'tvals': waveform.xvals('Time'),
        }
        
        
        #if not self.ctrls['parallel'].isChecked():
        output = processEventFits(events, startEvent=0, stopEvent=len(events), opts=opts)
        guesses = output['guesses']
        eventData = output['eventData']
        indexes = output['indexes']
        xVals = output['xVals']
        yVals = output['yVals']
        output = output['output']
        #else:
            #print "parallel:", self.pool, self.poolSize
            #results = []
            #nProcesses = self.ctrls['nProcesses'].value()
            #evPerProcess = int(len(events) / nProcesses)
            #start = 0
            #for i in range(nProcesses):
                #stop = start + evPerProcess
                #if stop > len(events):
                    #stop = len(events)
                #args = (events, start, stop, opts)
                #results.append(self.pool.apply_async(processEventFits, args))
                #print "started process", start, stop
                #start = stop
            #data = []
            #guesses = []
            #eventData = []
            #indexes = []
            #xVals = []
            #yVals = []
            #for res in results:  ## reconstruct results here
                #print "getting result", res
                #output = res.get(10)
                #data.append(output['output'])
                #guesses.extend(output['guesses'])
                #eventData.extend(output['eventData'])
                #indexes.extend(output['indexes'])
                #xVals.extend(output['xVals'])
                #yVals.extend(output['yVals'])
            #output = np.concatenate(data)
            
        for i in range(len(indexes)):            
            if display and self.plot.isConnected():
                if self.ctrls['plotFits'].isChecked():
                    item = pg.PlotDataItem(x=xVals[i], y=yVals[i], pen=(0, 0, 255), clickable=True)
                    item.setZValue(100)
                    self.plotItems.append(item)
                    item.eventIndex = indexes[i]
                    item.sigClicked.connect(self.fitClicked)
                    item.deleted = False
                if self.ctrls['plotGuess'].isChecked():
                    item2 = pg.PlotDataItem(x=xVals[i], y=functions.pspFunc(guesses[i], xVals[i]), pen=(255, 0, 0))
                    item2.setZValue(100)
                    self.plotItems.append(item2)
                if self.ctrls['plotEvents'].isChecked():
                    item2 = pg.PlotDataItem(x=xVals[i], y=eventData[i], pen=(0, 255, 0))
                    item2.setZValue(100)
                    self.plotItems.append(item2)
                #plot = self.plot.connections().keys()[0].node().getPlot()
                #plot.addItem(item)
            
        self.outputData = output
        return {'output': output, 'plot': self.plotItems}

    def deleteSelected(self):
        item = self.selectedFit
        d = not item.deleted
        if d:
            self.deletedFits.append(item.eventIndex)
            self.selectedFit.setPen((100, 0, 0))
        else:
            self.deletedFits.remove(item.eventIndex)
            self.selectedFit.setPen((0, 0, 255))
        item.deleted = d
            
        inds = np.ones(len(self.outputData), dtype=bool)
        inds[self.deletedFits] = False
        self.setOutput(output=self.outputData[inds], plot=self.plotItems)
        


    ## Intercept keypresses on any plot that is connected.
    def connected(self, local, remote):
        if local is self.plot:
            self.filterPlot(remote.node())
            remote.node().sigPlotChanged.connect(self.filterPlot)
        CtrlNode.connected(self, local, remote)
        
    def disconnected(self, local, remote):
        if local is self.plot:
            self.filterPlot(remote.node(), install=False)<commit_msg>vcn analysis update - added function for generating frequency column in map_sites<commit_after># -*- coding: utf-8 -*-

from pyqtgraph.flowchart.library.common import *
import functions
import numpy as np
import scipy
#from pyqtgraph import graphicsItems
import pyqtgraph as pg
import metaarray
#import pyqtgraph.CheckTable as CheckTable
from collections import OrderedDict

class EventFitter(CtrlNode):
    """Takes a waveform and event list as input, returns extra information about each event.
    Optionally performs an exponential reconvolution before measuring each event.
    Plots fits of reconstructed events if the plot output is connected."""
    nodeName = "EventFitter"
    uiTemplate = [
        ('multiFit', 'check', {'value': False}),
        #('parallel', 'check', {'value': False}),
        #('nProcesses', 'spin', {'value': 1, 'min': 1, 'int': True}),
        ('plotFits', 'check', {'value': True}),
        ('plotGuess', 'check', {'value': False}),
        ('plotEvents', 'check', {'value': False}),
    ]
    
    
    def __init__(self, name):
        CtrlNode.__init__(self, name, terminals={
            'waveform': {'io': 'in'},
            'events': {'io': 'in'},
            'output': {'io': 'out'},
            'plot':  {'io': 'out'}
        })
        self.plotItems = []
        self.selectedFit = None
        self.deletedFits = []
        self.pool = None  ## multiprocessing pool
        self.poolSize = 0
        #self.ctrls['parallel'].toggled.connect(self.setupPool)
        #self.ctrls['nProcesses'].valueChanged.connect(self.setupPool)
    
    #def setupPool(self):
        #import multiprocessing as mp
        #if self.ctrls['parallel'].isChecked():
            #nProc = self.ctrls['nProcesses'].value()
            #if self.pool is not None and self.poolSize != nProc:
                #self.pool.terminate()
                #self.pool = None
            #if self.pool is None:
                #self.pool = mp.Pool(processes=nProc)
                #self.poolSize = nProc
        #else:
            #if self.pool is not None:
                #self.pool.terminate()
                #self.pool = None
    
    def process(self, waveform, events, display=True):
        self.deletedFits = []
        for item in self.plotItems:
            try:
                item.sigClicked.disconnect(self.fitClicked)
            except:
                pass
        self.plotItems = []
        
        tau = waveform.infoCopy(-1).get('expDeconvolveTau', None)
        dt = waveform.xvals(0)[1] - waveform.xvals(0)[0]
        opts = {
            'dt': dt, 'tau': tau, 'multiFit': self.ctrls['multiFit'].isChecked(),
            'waveform': waveform.view(np.ndarray),
            'tvals': waveform.xvals('Time'),
        }
        
        
        #if not self.ctrls['parallel'].isChecked():
        output = processEventFits(events, startEvent=0, stopEvent=len(events), opts=opts)
        guesses = output['guesses']
        eventData = output['eventData']
        indexes = output['indexes']
        xVals = output['xVals']
        yVals = output['yVals']
        output = output['output']
        #else:
            #print "parallel:", self.pool, self.poolSize
            #results = []
            #nProcesses = self.ctrls['nProcesses'].value()
            #evPerProcess = int(len(events) / nProcesses)
            #start = 0
            #for i in range(nProcesses):
                #stop = start + evPerProcess
                #if stop > len(events):
                    #stop = len(events)
                #args = (events, start, stop, opts)
                #results.append(self.pool.apply_async(processEventFits, args))
                #print "started process", start, stop
                #start = stop
            #data = []
            #guesses = []
            #eventData = []
            #indexes = []
            #xVals = []
            #yVals = []
            #for res in results:  ## reconstruct results here
                #print "getting result", res
                #output = res.get(10)
                #data.append(output['output'])
                #guesses.extend(output['guesses'])
                #eventData.extend(output['eventData'])
                #indexes.extend(output['indexes'])
                #xVals.extend(output['xVals'])
                #yVals.extend(output['yVals'])
            #output = np.concatenate(data)
            
        for i in range(len(indexes)):            
            if display and self.plot.isConnected():
                if self.ctrls['plotFits'].isChecked():
                    item = pg.PlotDataItem(x=xVals[i], y=yVals[i], pen=(0, 0, 255), clickable=True)
                    item.setZValue(100)
                    self.plotItems.append(item)
                    item.eventIndex = indexes[i]
                    item.sigClicked.connect(self.fitClicked)
                    item.deleted = False
                if self.ctrls['plotGuess'].isChecked():
                    item2 = pg.PlotDataItem(x=xVals[i], y=functions.pspFunc(guesses[i], xVals[i]), pen=(255, 0, 0))
                    item2.setZValue(100)
                    self.plotItems.append(item2)
                if self.ctrls['plotEvents'].isChecked():
                    item2 = pg.PlotDataItem(x=xVals[i], y=eventData[i], pen=(0, 255, 0))
                    item2.setZValue(100)
                    self.plotItems.append(item2)
                #plot = self.plot.connections().keys()[0].node().getPlot()
                #plot.addItem(item)
            
        self.outputData = output
        return {'output': output, 'plot': self.plotItems}

    def deleteSelected(self):
        item = self.selectedFit
        d = not item.deleted
        if d:
            self.deletedFits.append(item.eventIndex)
            self.selectedFit.setPen((100, 0, 0))
        else:
            self.deletedFits.remove(item.eventIndex)
            self.selectedFit.setPen((0, 0, 255))
        item.deleted = d
            
        inds = np.ones(len(self.outputData), dtype=bool)
        inds[self.deletedFits] = False
        self.setOutput(output=self.outputData[inds], plot=self.plotItems)
        


    ## Intercept keypresses on any plot that is connected.
    def connected(self, local, remote):
        if local is self['plot']:
            self.filterPlot(remote.node())
            remote.node().sigPlotChanged.connect(self.filterPlot)
        CtrlNode.connected(self, local, remote)
        
    def disconnected(self, local, remote):
        if local is self.plot:
            self.filterPlot(remote.node(), install=False)
{'fraction_non_alphanumeric': 0.13117012321114258, 'fraction_numerical': 0.00826509527818168, 'mean_word_length': 1.8445798868088812, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5261762', 'n_tokens_mistral': 3909, 'n_tokens_neox': 3710, 'n_words': 805}