<commit_before># @Author: <NAME>
# @Date:   Nov 10, 2014
# @Filename: batch_decomposition.py
# @Last modified by:   riener
# @Last modified time: 2019-03-16T15:59:58+01:00

import pickle
import multiprocessing
import signal
import numpy as np
# from . import AGD_decomposer
from .gp import GaussianDecomposer
from tqdm import tqdm
from concurrent.futures import ProcessPoolExecutor, as_completed

# BUG FIXED:  UnboundLocalError: local variable 'result' referenced before assignment
# Caused by using more threads than spectra.


def init_worker():
    """Worker initializer to ignore Keyboard interrupt."""
    signal.signal(signal.SIGINT, signal.SIG_IGN)
    signal.signal(signal.SIGPIPE, signal.SIG_DFL)


def init(*args):
    global agd_object, science_data_path, ilist, agd_data
    if args:
        [agd_object, agd_data, ilist] = args[0]
    else:
        [agd_object, science_data_path, ilist] = pickle.load(
            open('batchdecomp_temp.pickle', 'rb'), encoding='latin1')
        agd_data = pickle.load(open(science_data_path, 'rb'), encoding='latin1')
    if ilist == None:
        ilist = np.arange(len(agd_data['data_list']))


def decompose_one(i):
    print '   ---->  ', i 
    result = GaussianDecomposer.decompose(agd_object, 
                                          agd_data['x_values'][i], 
                                          agd_data['data_list'][i], 
                                          agd_data['errors'][i])
    return result<commit_msg>Fixed silly lil print typo<commit_after># @Author: <NAME>
# @Date:   Nov 10, 2014
# @Filename: batch_decomposition.py
# @Last modified by:   riener
# @Last modified time: 2019-03-16T15:59:58+01:00

import pickle
import multiprocessing
import signal
import numpy as np
# from . import AGD_decomposer
from .gp import GaussianDecomposer
from tqdm import tqdm
from concurrent.futures import ProcessPoolExecutor, as_completed

# BUG FIXED:  UnboundLocalError: local variable 'result' referenced before assignment
# Caused by using more threads than spectra.


def init_worker():
    """Worker initializer to ignore Keyboard interrupt."""
    signal.signal(signal.SIGINT, signal.SIG_IGN)
    signal.signal(signal.SIGPIPE, signal.SIG_DFL)


def init(*args):
    global agd_object, science_data_path, ilist, agd_data
    if args:
        [agd_object, agd_data, ilist] = args[0]
    else:
        [agd_object, science_data_path, ilist] = pickle.load(
            open('batchdecomp_temp.pickle', 'rb'), encoding='latin1')
        agd_data = pickle.load(open(science_data_path, 'rb'), encoding='latin1')
    if ilist == None:
        ilist = np.arange(len(agd_data['data_list']))


def decompose_one(i):
    print("   ---->  ", i) 
    result = GaussianDecomposer.decompose(agd_object, 
                                          agd_data['x_values'][i], 
                                          agd_data['data_list'][i], 
                                          agd_data['errors'][i])
    return result
{'fraction_non_alphanumeric': 0.1081989247311828, 'fraction_numerical': 0.018145161290322582, 'mean_word_length': 2.8864229765013056, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 12, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5191743', 'n_tokens_mistral': 980, 'n_tokens_neox': 925, 'n_words': 227}