<commit_before>"""
A collection of tools for timeseries

:author: <NAME> & <NAME>
:contact: pierregm_at_uga_dot_edu - mattknow_ca_at_hotmail_dot_com
:version: $Id$
"""
__author__ = "<NAME> & <NAME> ($Author$)"
__version__ = '1.0'
__revision__ = "$Revision$"
__date__     = '$Date$'

import numpy
import numpy.core.numeric as numeric

from scipy.interpolate import fitpack

import maskedarray as MA
from maskedarray import masked, nomask, getmask

#####---------------------------------------------------------------------------
#---- --- Generic functions ---
#####---------------------------------------------------------------------------
def first_unmasked_val(a):
    "Returns the first unmasked value in a 1d maskedarray."
    (i,j) = MA.extras.flatnotmasked_edges(a)
    return a[i]

def last_unmasked_val(a):
    "Returns the last unmasked value in a 1d maskedarray."
    (i,j) = MA.extras.flatnotmasked_edges(a)
    return a[j]

def reverse_dict(d):
    "Reverses the keys and values of a dictionary."
    alt = []
    tmp = [alt.extend([(w,k) for w in v]) for (k,v) in d.iteritems()]
    return dict(alt)



#####---------------------------------------------------------------------------
#---- --- Option conversion ---
#####---------------------------------------------------------------------------
fmtobs_dict = {'UNDEF': ['UNDEF','UNDEFINED'],
               'BEGINNING': ['BEGIN','BEGINNING'],
               'ENDING': ['END','ENDING'],
               'AVERAGED': ['AVERAGE','AVERAGE','MEAN'],
               'SUMMED': ['SUM','SUMMED'],
               'MAXIMUM': ['MAX','MAXIMUM','HIGH'],
               'MINIMUM': ['MIN','MINIMUM','LOW']}

obs_dict = {"UNDEFINED":None,
            "UNDEF":None,
            "BEGIN": first_unmasked_val,
            "BEGINNING": first_unmasked_val,
            "END": last_unmasked_val,
            "ENDING": last_unmasked_val,
            "AVERAGED": MA.average,
            "AVERAGE": MA.average,
            "MEAN": MA.average,
            "SUMMED": MA.sum,
            "SUM": MA.sum,<commit_msg>changed undef to undefined in fmtobs_dict key<commit_after>"""
A collection of tools for timeseries

:author: <NAME> & <NAME>
:contact: pierregm_at_uga_dot_edu - mattknow_ca_at_hotmail_dot_com
:version: $Id$
"""
__author__ = "<NAME> & <NAME> ($Author$)"
__version__ = '1.0'
__revision__ = "$Revision$"
__date__     = '$Date$'

import numpy
import numpy.core.numeric as numeric

from scipy.interpolate import fitpack

import maskedarray as MA
from maskedarray import masked, nomask, getmask

#####---------------------------------------------------------------------------
#---- --- Generic functions ---
#####---------------------------------------------------------------------------
def first_unmasked_val(a):
    "Returns the first unmasked value in a 1d maskedarray."
    (i,j) = MA.extras.flatnotmasked_edges(a)
    return a[i]

def last_unmasked_val(a):
    "Returns the last unmasked value in a 1d maskedarray."
    (i,j) = MA.extras.flatnotmasked_edges(a)
    return a[j]

def reverse_dict(d):
    "Reverses the keys and values of a dictionary."
    alt = []
    tmp = [alt.extend([(w,k) for w in v]) for (k,v) in d.iteritems()]
    return dict(alt)



#####---------------------------------------------------------------------------
#---- --- Option conversion ---
#####---------------------------------------------------------------------------
fmtobs_dict = {'UNDEFINED': ['UNDEF','UNDEFINED'],
               'BEGINNING': ['BEGIN','BEGINNING'],
               'ENDING': ['END','ENDING'],
               'AVERAGED': ['AVERAGE','AVERAGE','MEAN'],
               'SUMMED': ['SUM','SUMMED'],
               'MAXIMUM': ['MAX','MAXIMUM','HIGH'],
               'MINIMUM': ['MIN','MINIMUM','LOW']}

obs_dict = {"UNDEFINED":None,
            "UNDEF":None,
            "BEGIN": first_unmasked_val,
            "BEGINNING": first_unmasked_val,
            "END": last_unmasked_val,
            "ENDING": last_unmasked_val,
            "AVERAGED": MA.average,
            "AVERAGE": MA.average,
            "MEAN": MA.average,
            "SUMMED": MA.sum,
            "SUM": MA.sum,
{'fraction_non_alphanumeric': 0.28799612778315586, 'fraction_numerical': 0.001936108422071636, 'mean_word_length': 1.6855100714749838, 'pattern_counts': {'":': 22, '<': 11, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7224847', 'n_tokens_mistral': 1283, 'n_tokens_neox': 1226, 'n_words': 281}