<commit_before>#!/usr/bin/env python

import os
from time import time
from datetime import datetime
from collections import OrderedDict
import fitsio
import numpy as np

from astropy.stats import sigma_clip

# just translates the kwargs
def array_clip(arr,axis=None,**kwargs):
	arr = sigma_clip(arr,axis=axis,
	                 iters=kwargs.get('clip_iters',2),
	                 sig=kwargs.get('clip_sig',2.5),
	                 cenfunc=kwargs.get('clip_cenfunc',np.ma.mean))
	return arr

def array_stats(arr,axis=None,method='median',clip=True,rms=False,
                retArray=False,**kwargs):
	if clip:
		arr = array_clip(arr,axis=axis,**kwargs)
	if method=='median':
		val = np.ma.median(arr,axis=axis)
	elif method=='mean':
		val = np.ma.mean(arr,axis=axis)
	elif method=='mode':
		val = 3*np.ma.median(arr,axis=axis) - 2*np.ma.mean(arr,axis=axis)
	else:
		raise ValueError('array stats method %s unrecognized' % method)
	if axis==None:
		# median returns an array but mean doesn't???
		try:
			val = val.compressed()
		except:
			pass
		val = val.squeeze()
	rv = [val]
	if rms:
		_rms = np.ma.std(arr,axis=axis)
		rv.append(_rms)
	if retArray:<commit_msg>handle changes in sigma_clip<commit_after>#!/usr/bin/env python

import os
from time import time
from datetime import datetime
from collections import OrderedDict
import fitsio
import numpy as np

from astropy.stats import sigma_clip

# just translates the kwargs
def array_clip(arr,axis=None,**kwargs):
	# for some reason in newer version of astropy (>1.1) axis=-1 
	# no longer works ...
	if axis < 0:
		axis = len(arr.shape) + axis
	arr = sigma_clip(arr,axis=axis,
	                 sigma=kwargs.get('clip_sig',2.5),
	                 iters=kwargs.get('clip_iters',2),
	                 cenfunc=kwargs.get('clip_cenfunc',np.ma.mean))
	return arr

def array_stats(arr,axis=None,method='median',clip=True,rms=False,
                retArray=False,**kwargs):
	if clip:
		arr = array_clip(arr,axis=axis,**kwargs)
	if method=='median':
		val = np.ma.median(arr,axis=axis)
	elif method=='mean':
		val = np.ma.mean(arr,axis=axis)
	elif method=='mode':
		val = 3*np.ma.median(arr,axis=axis) - 2*np.ma.mean(arr,axis=axis)
	else:
		raise ValueError('array stats method %s unrecognized' % method)
	if axis==None:
		# median returns an array but mean doesn't???
		try:
			val = val.compressed()
		except:
			pass
		val = val.squeeze()
	rv = [val]
	if rms:
		_rms = np.ma.std(arr,axis=axis)
		rv.append(_rms)
	if retArray:
{'fraction_non_alphanumeric': 0.1381260096930533, 'fraction_numerical': 0.005654281098546042, 'mean_word_length': 3.7361376673040154, 'pattern_counts': {'":': 0, '<': 4, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3578838', 'n_tokens_mistral': 946, 'n_tokens_neox': 865, 'n_words': 205}