<commit_before># masking_reshape_functions.py
# this file provides a way to mask data, then reduce it's dimensions 
# (and then create the correct output after analysis is done on 1d to 2d data)
import numpy as np
import itertools
import scipy.ndimage
from scipy.ndimage.filters import gaussian_filter
import matplotlib
matplotlib.use('Agg')
import matplotlib.pyplot as plt
import nibabel as nib
import os
import sys


def masking_reshape_start(data, mask):
	"""
	takes a 3 or 4d data and utilizes a mask to return a 1 or 2d reshaped output

	Input:
	------
	data: 3d *or* 4d np.array  (x,y,z) or (x,y,z,t) shape
	mask: a 3d np array  (x,y,z) shape, with values 0s and 1s (1 desired, 0 remove)

	Returns:
	--------
	reshaped: a 1d *or* 2d np.array (connected to 3d or 4d "data" input)

	"""
	assert(len(data.shape) == 3 or len(data.shape) == 4)

	if len(data.shape) == 3:
		data_1d = np.ravel(data)
		reshaped = data_1d[np.ravel(mask == 1)]

	if len(data.shape) == 4:
		data_2d = data.reshape((-1, data.shape[-1]))
		reshaped = data_2d[np.ravel(mask == 1), :]
	return reshaped



def masking_reshape_end(data_small, mask, off_value=0):
	"""
	takes a 1d input, utilizes a mask to convert into 3d output ()

	Notes:
	------
	mask must have same number of ones as the data_small.shape[0]

	Input:
	------
	data_small: a 1d np.array
	mask:       a 3d np array  (x,y,z) shape, with values 0s and 1s (1 desired, 0 remove), see notes
	off_value:  the value to be replaced for the non-on values of the mask

<commit_msg>correction to remove warning problem<commit_after># masking_reshape_functions.py
# this file provides a way to mask data, then reduce it's dimensions 
# (and then create the correct output after analysis is done on 1d to 2d data)
import numpy as np
import itertools
import scipy.ndimage
from scipy.ndimage.filters import gaussian_filter
import matplotlib
matplotlib.use('Agg')
import matplotlib.pyplot as plt
import nibabel as nib
import os
import sys


def masking_reshape_start(data, mask):
	"""
	takes a 3 or 4d data and utilizes a mask to return a 1 or 2d reshaped output

	Input:
	------
	data: 3d *or* 4d np.array  (x,y,z) or (x,y,z,t) shape
	mask: a 3d np array  (x,y,z) shape, with values 0s and 1s (1 desired, 0 remove)

	Returns:
	--------
	reshaped: a 1d *or* 2d np.array (connected to 3d or 4d "data" input)

	"""
	assert(len(data.shape) == 3 or len(data.shape) == 4)

	mask_1d=np.ravel(mask)
	b_mask_1d = (mask_1d==1)


	if len(data.shape) == 3:
		data_1d = np.ravel(data)
		reshaped = data_1d[b_mask_1d]

	if len(data.shape) == 4:
		data_2d = data.reshape((-1, data.shape[-1]))
		reshaped = data_2d[b_mask_1d, :]
	return reshaped



def masking_reshape_end(data_small, mask, off_value=0):
	"""
	takes a 1d input, utilizes a mask to convert into 3d output ()

	Notes:
	------
	mask must have same number of ones as the data_small.shape[0]

	Input:
	------
	data_small: a 1d np.array
	mask:       a 3d np array  (x,y,z) shape, with values 0s and 1s (1 desired, 0 remove), see notes
	off_value:  the value to be replaced for the non-on values of the mask


{'fraction_non_alphanumeric': 0.11464761286131861, 'fraction_numerical': 0.026632023384215654, 'mean_word_length': 3.6246246246246248, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5826607', 'n_tokens_mistral': 1243, 'n_tokens_neox': 1130, 'n_words': 413}