<commit_before>#!/usr/bin/env python
#
#   Copyright 2016 <NAME>
#
#   Licensed under the Apache License, Version 2.0 (the "License");
#   you may not use this file except in compliance with the License.
#   You may obtain a copy of the License at
#
#       http://www.apache.org/licenses/LICENSE-2.0
#
#   Unless required by applicable law or agreed to in writing, software
#   distributed under the License is distributed on an "AS IS" BASIS,
#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#   See the License for the specific language governing permissions and
#   limitations under the License.
#
#
# $Author: frederic $
# $Date: 2016/07/11 14:50:43 $
# $Id: rapidtide,v 1.161 2016/07/11 14:50:43 frederic Exp $
#
#
#

from __future__ import print_function, division

import numpy as np

import rapidtide.fit as tide_fit
import rapidtide.multiproc as tide_multiproc
import rapidtide.util as tide_util


def _procOneVoxelGLM(vox, lagtc, inittc, rt_floatset=np.float64, rt_floattype='float64'):
    thefit, R = tide_fit.mlregress(lagtc, inittc)
    fitcoff = rt_floatset(thefit[0, 1])
    datatoremove = rt_floatset(fitcoff * lagtc)
    return vox, rt_floatset(thefit[0, 0]), rt_floatset(R), rt_floatset(R * R), fitcoff, \
           rt_floatset(thefit[0, 1] / thefit[0, 0]), datatoremove, rt_floatset(inittc - datatoremove)


def glmpass(numspatiallocs, reportstep, fmri_data, threshval, lagtc, optiondict, meanvalue, rvalue, r2value, fitcoff,
            fitNorm, datatoremove, filtereddata):
    inputshape = np.shape(fmri_data)
    if optiondict['nprocs'] > 1:
        # define the consumer function here so it inherits most of the arguments
        def GLM_consumer(inQ, outQ):
            while True:
                try:
                    # get a new message
                    val = inQ.get()

                    # this is the 'TERM' signal
                    if val is None:
                        break

                    # process and send the data
                    outQ.put(_procOneVoxelGLM(val,
                                              lagtc[val, :],
                                              fmri_data[val,
                                              optiondict['addedskip']:],
                                              rt_floatset=np.float64,
                                              rt_floattype='float64'))

                except Exception as e:
                    print("error!", e)
                    break

        data_out = tide_multiproc.run_multiproc(GLM_consumer,
                                                inputshape, None,
                                                nprocs=optiondict['nprocs'],
                                                showprogressbar=True,
                                                chunksize=optiondict['mp_chunksize'])

        # unpack the data
        volumetotal = 0
        for voxel in data_out:
            meanvalue[voxel[0]] = voxel[1]
            rvalue[voxel[0]] = voxel[2]
            r2value[voxel[0]] = voxel[3]
            fitcoff[voxel[0]] = voxel[4]
            fitNorm[voxel[0]] = voxel[5]
            datatoremove[voxel[0], :] = voxel[6]
            filtereddata[voxel[0], :] = voxel[7]
            volumetotal += 1
        data_out = []
    else:
        volumetotal = 0
        for vox in range(0, numspatiallocs):
            if (vox % reportstep == 0 or vox == numspatiallocs - 1) and optiondict['showprogressbar']:
                tide_util.progressbar(vox + 1, numspatiallocs, label='Percent complete')
            inittc = fmri_data[vox, optiondict['addedskip']:].copy()
            if np.mean(inittc) >= threshval:
                dummy, meanvalue[vox], rvalue[vox], r2value[vox], fitcoff[vox], fitNorm[vox], datatoremove[vox], \
                filtereddata[vox] = \
                    _procOneVoxelGLM(vox,
                                     lagtc[vox, :],
                                     inittc,
                                     rt_floatset=np.float64,
                                     rt_floattype='float64')
                volumetotal += 1

    return volumetotal<commit_msg>Fixed some internal arguments, now properly use mask in multiprocessor mode<commit_after>#!/usr/bin/env python
#
#   Copyright 2016 <NAME>
#
#   Licensed under the Apache License, Version 2.0 (the "License");
#   you may not use this file except in compliance with the License.
#   You may obtain a copy of the License at
#
#       http://www.apache.org/licenses/LICENSE-2.0
#
#   Unless required by applicable law or agreed to in writing, software
#   distributed under the License is distributed on an "AS IS" BASIS,
#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#   See the License for the specific language governing permissions and
#   limitations under the License.
#
#
# $Author: frederic $
# $Date: 2016/07/11 14:50:43 $
# $Id: rapidtide,v 1.161 2016/07/11 14:50:43 frederic Exp $
#
#
#

from __future__ import print_function, division

import numpy as np

import rapidtide.fit as tide_fit
import rapidtide.multiproc as tide_multiproc
import rapidtide.util as tide_util


def _procOneVoxelGLM(vox, lagtc, inittc, rt_floatset=np.float64, rt_floattype='float64'):
    thefit, R = tide_fit.mlregress(lagtc, inittc)
    fitcoff = rt_floatset(thefit[0, 1])
    datatoremove = rt_floatset(fitcoff * lagtc)
    return vox, rt_floatset(thefit[0, 0]), rt_floatset(R), rt_floatset(R * R), fitcoff, \
           rt_floatset(thefit[0, 1] / thefit[0, 0]), datatoremove, rt_floatset(inittc - datatoremove)


def glmpass(numspatiallocs, reportstep, fmri_data, threshval, lagtc, optiondict, meanvalue, rvalue, r2value, fitcoff,
            fitNorm, datatoremove, filtereddata):
    inputshape = np.shape(fmri_data)
    themask = np.where(np.mean(fmri_data, axis=1) > threshval, 1, 0)
    if optiondict['nprocs'] > 1:
        # define the consumer function here so it inherits most of the arguments
        def GLM_consumer(inQ, outQ):
            while True:
                try:
                    # get a new message
                    val = inQ.get()

                    # this is the 'TERM' signal
                    if val is None:
                        break

                    # process and send the data
                    outQ.put(_procOneVoxelGLM(val,
                                              lagtc[val, :],
                                              fmri_data[val, :],
                                              rt_floatset=np.float64,
                                              rt_floattype='float64'))

                except Exception as e:
                    print("error!", e)
                    break

        data_out = tide_multiproc.run_multiproc(GLM_consumer,
                                                inputshape, themask,
                                                nprocs=optiondict['nprocs'],
                                                showprogressbar=True,
                                                chunksize=optiondict['mp_chunksize'])

        # unpack the data
        volumetotal = 0
        for voxel in data_out:
            meanvalue[voxel[0]] = voxel[1]
            rvalue[voxel[0]] = voxel[2]
            r2value[voxel[0]] = voxel[3]
            fitcoff[voxel[0]] = voxel[4]
            fitNorm[voxel[0]] = voxel[5]
            datatoremove[voxel[0], :] = voxel[6]
            filtereddata[voxel[0], :] = voxel[7]
            volumetotal += 1

        del data_out
    else:
        volumetotal = 0
        for vox in range(0, numspatiallocs):
            if (vox % reportstep == 0 or vox == numspatiallocs - 1) and optiondict['showprogressbar']:
                tide_util.progressbar(vox + 1, numspatiallocs, label='Percent complete')
            inittc = fmri_data[vox, optiondict['addedskip']:].copy()
            if themask[vox] > 0:
                dummy, \
                meanvalue[vox],\
                rvalue[vox], \
                r2value[vox], \
                fitcoff[vox], \
                fitNorm[vox], \
                datatoremove[vox], \
                filtereddata[vox] = \
                    _procOneVoxelGLM(vox,
                                     lagtc[vox, :],
                                     inittc,
                                     rt_floatset=np.float64,
                                     rt_floattype='float64')
                volumetotal += 1

    return volumetotal
{'fraction_non_alphanumeric': 0.09084473808400188, 'fraction_numerical': 0.020764511562057573, 'mean_word_length': 1.6075053829590895, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 10, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 42, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '618073', 'n_tokens_mistral': 2625, 'n_tokens_neox': 2419, 'n_words': 609}