<commit_before>from dispel4py.core import GenericPE
from dispel4py.workflow_graph import WorkflowGraph
from dispel4py.base import BasePE, IterativePE, ConsumerPE, create_iterative_chain

import numpy as np
import time
import os, shutil
import traceback
from tc_cross_correlation.xcorr import xcorrf, xcorrf_noFFT
from  distutils.dir_util  import create_tree as mkadir
import matplotlib as mpl
mpl.use('Agg')
import matplotlib.pyplot as plt
import matplotlib.backends
from obspy import read
import codecs, json
import pickle

ROOT_DIR = './tc_cross_correlation/OUTPUT/'
starttime='2015-04-06T06:00:00.000'

class Product(GenericPE):
    def __init__(self):
        GenericPE.__init__(self)
        self._add_output('output', tuple_type=['counter1', 'counter2', 'station1', 'station2'])
import matplotlib as mpl
mpl.use('Agg')
import matplotlib.pyplot as plt
import matplotlib.backends
from obspy import read
import codecs, json
import pickle

ROOT_DIR = './tc_cross_correlation/OUTPUT/'
starttime='2015-04-06T06:00:00.000'

class Product(GenericPE):
    def __init__(self):
        GenericPE.__init__(self)
        self._add_output('output', tuple_type=['counter1', 'counter2', 'station1', 'station2'])
    def process(self, inputs):
        store = []
        for dir in os.listdir(ROOT_DIR + 'DATA/'+ starttime):
            for f in os.listdir(ROOT_DIR+'/DATA/'+ starttime +'/'+ dir):
               file=ROOT_DIR+'DATA/'+ starttime + '/'+ dir+'/'+f
               self.log("open file %s" % file)
               str1_array= np.load(file)
               str1 = pickle.dumps(str1_array)
               index = len(store)
               for i in range(index):
                  self.write('output',[i, index, store[i], str1])
               store.append(str1)


class Xcorr(IterativePE):
    def __init__(self):
        IterativePE.__init__(self)
        self._add_input ('input')
        self._add_output('output', tuple_type=['counter1','counter2', 'xcorr'])
    def _process(self, data):                                                                                                                        
        str1=pickle.lodas(data[2])                                                                                                                   
        str2=pickle.loads(data[3])                                                                                                                   
        try:                                                                                                                                         
            xcorr1 = xcorrf_noFFT(str1, str2, 5000)                                                                                                  
            return [data[0], data[1], xcorr1]                                                                                                        
        except:                                                                                                                                      
            self.log('error in %s_%s xcorr' % (data[0], data[1]))                                                                                    
            self.log(traceback.format_exc())                                                                                                         
                                                                                                                                                     <commit_msg>xcorr working<commit_after>from dispel4py.core import GenericPE
from dispel4py.workflow_graph import WorkflowGraph
from dispel4py.base import BasePE, IterativePE, ConsumerPE, create_iterative_chain

import numpy as np
import time
import os, shutil
import traceback
from tc_cross_correlation.xcorr import xcorrf, xcorrf_noFFT
from  distutils.dir_util  import create_tree as mkadir
import matplotlib as mpl
mpl.use('Agg')
import matplotlib.pyplot as plt
import matplotlib.backends
from obspy import read
import codecs, json
import pickle

ROOT_DIR = './tc_cross_correlation/OUTPUT/'
starttime='2015-04-06T06:00:00.000'

class Product(GenericPE):
    def __init__(self):
        GenericPE.__init__(self)
        self._add_output('output', tuple_type=['counter1', 'counter2', 'station1', 'station2'])
import matplotlib as mpl
mpl.use('Agg')
import matplotlib.pyplot as plt
import matplotlib.backends
from obspy import read
import codecs, json
import pickle

ROOT_DIR = './tc_cross_correlation/OUTPUT/'
starttime='2015-04-06T06:00:00.000'

class Product(GenericPE):
    def __init__(self):
        GenericPE.__init__(self)
        self._add_output('output', tuple_type=['counter1', 'counter2', 'station1', 'station2'])
    def process(self, inputs):
        store = []
        for dir in os.listdir(ROOT_DIR + 'DATA/'+ starttime):
            for f in os.listdir(ROOT_DIR+'/DATA/'+ starttime +'/'+ dir):
               file=ROOT_DIR+'DATA/'+ starttime + '/'+ dir+'/'+f
               self.log("open file %s" % file)
               str1_array= np.load(file)
               str1 = pickle.dumps(str1_array)
               index = len(store)
               for i in range(index):
                  self.write('output',[i, index, store[i], str1])
               store.append(str1)


class Xcorr(IterativePE):
    def __init__(self):
        IterativePE.__init__(self)
        self._add_input ('input')
        self._add_output('output', tuple_type=['counter1','counter2', 'xcorr'])
    def _process(self, data):                                                                                                                        
        str1=pickle.loads(data[2])                                                                                                                   
        str2=pickle.loads(data[3])                                                                                                                   
        try:                                                                                                                                         
            xcorr1 = xcorrf_noFFT(str1, str2, 5000)                                                                                                  
            return [data[0], data[1], xcorr1]                                                                                                        
        except:                                                                                                                                      
            self.log('error in %s_%s xcorr' % (data[0], data[1]))                                                                                    
            self.log(traceback.format_exc())                                                                                                         
                                                                                                                                                     
{'fraction_non_alphanumeric': 0.08182080092192452, 'fraction_numerical': 0.01959089599539038, 'mean_word_length': 1.024198250728863, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 96, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2630667', 'n_tokens_mistral': 1607, 'n_tokens_neox': 1514, 'n_words': 347}