<commit_before>#!/usr/bin/env python
"""
  lib.py

  State Estimation and Analysis for PYthon

  Library of utilities for general seapy module, imported into the namespace
  when importing the seapy module

  Written by <NAME> on 10/18/13
  Copyright (c)2016 University of Hawaii under the BSD-License.
"""


import numpy as np
from scipy import ndimage
import os
import re
import datetime
import itertools


secs2day = 1.0 / 86400.0
default_epoch = datetime.datetime(2000, 1, 1)
_default_timeref = "days since " + default_epoch.strftime("%Y-%m-%m %H:%M:%S")


def adddim(fld, size=1):
    """
    replicate a field and add a new first dimension with given size

    Parameters
    ----------
    fld : array_like
        Input field.
    size : int, optional
        Size of additional first dimension

    Returns
    -------
    fld : array

    Examples
    --------
    >>> a=np.array([4, 5, 6, 7])
    >>> a.shape
    (4,)
    >>> b = seapy.adddim(a, 2)
    >>> b.shape
    (2, 4)
    >>> b
    array([[4, 5, 6, 7],
           [4, 5, 6, 7]])

    """
    fld = np.atleast_1d(fld)
    s = np.ones(fld.ndim + 1).astype(int)
    s[0] = int(size)
    return np.tile(fld, s)


def chunker(seq, size):
    """
    Iterate over an iterable in 'chunks' of a given size

    Parameters
    ----------
    seq : iterable,
        The sequence to iterate over
    size : int,
        The number of items to be returned in each 'chunk'

    Returns
    -------
    chunk : seq,
        The items of the chunk to be iterated

    Examples
    --------
    >>> x = [0,3,4,7,9,10,12,14]
    >>> for i in chunker(x, 3):
    >>>     print(i)
    [0, 3, 4]
    [7, 9, 10]
    [12, 14]

    """
    return (seq[pos:pos + size] for pos in range(0, len(seq), size))


def smooth(data, ksize=3, kernel=None, copy=True):
    """
    Smooth the data field using a specified convolution kernel
    or a default averaging kernel.

    Parameters
    ----------
    data : masked array_like
        Input field.
    ksize : int, optional
        Size of square kernel
    kernel : ndarray, optional
        Define a convolution kernel. Default is averaging
    copy : bool, optional
        If true, a copy of input array is made

    Returns
    -------
    fld : masked array
    """
    fld = np.ma.array(data, copy=copy)
    mask = np.ma.getmaskarray(fld).copy()

    # Make sure ksize is odd
    ksize = int(ksize + 1) if int(ksize) % 2 == 0 else int(ksize)
    if fld.ndim > 3 or fld.ndim < 2:
        raise AttributeError("Can only convolve 2- or 3-D fields")
    if ksize < 3:
        raise ValueError("ksize must be greater than or equal to 3")

    if kernel is None:
        kernel = np.ones((ksize, ksize)) / (ksize * ksize)
    else:
        ksize = kernel.shape[0]

    # First, convole over any masked values
    fld = convolve_mask(fld, ksize=ksize, copy=False)

    # Next, perform the convolution
    if fld.ndim == 2:
        fld = ndimage.convolve(fld.data, kernel,
                               mode="reflect", cval=0.0)
    else:
        kernel = np.expand_dims(kernel, axis=3)
        fld = np.transpose(ndimage.convolve(
            fld.filled(0).transpose(1, 2, 0), kernel,
            mode="reflect", cval=0.0), (2, 0, 1))

    # Apply the initial mask
    return np.ma.array(fld, mask=mask)


def convolve_mask(data, ksize=3, kernel=None, copy=True):
    """
    Convolve data over the missing regions of a mask

    Parameters
    ----------
    data : masked array_like
        Input field.
    ksize : int, optional
        Size of square kernel
    kernel : ndarray, optional
        Define a convolution kernel. Default is averaging
    copy : bool, optional
        If true, a copy of input array is made

    Returns
    -------
    fld : masked array
    """
    fld = np.ma.array(data, copy=copy)
    if not copy:
        fld._sharedmask = False

    # Make sure ksize is odd
    ksize = int(ksize + 1) if int(ksize) % 2 == 0 else int(ksize)
    if fld.ndim > 3 or fld.ndim < 2:
        raise AttributeError("Can only convolve 2- or 3-D fields")
    if ksize < 3:
        raise ValueError("ksize must be greater than or equal to 3")

    if kernel is None:
        center = np.round(ksize / 2).astype(int)
        kernel = np.ones([ksize, ksize])
        kernel[center, center] = 0.0
    else:
        ksize = kernel.shape[0]

    # Convolve the mask
    msk = np.ma.getmaskarray(fld)
    if fld.ndim == 2:
        count = ndimage.convolve((~msk).view(np.int8), kernel,
                                 mode="constant", cval=0.0)
        nfld = ndimage.convolve(fld.data * (~msk).view(np.int8), kernel,
                                mode="constant", cval=0.0)
    else:
        kernel = np.expand_dims(kernel, axis=3)
        count = np.transpose(ndimage.convolve(
            (~msk).view(np.int8).transpose(1, 2, 0), kernel,
            mode="constant", cval=0.0), (2, 0, 1))
        nfld = np.transpose(ndimage.convolve(
            (fld.data * (~msk).view(np.int8)).transpose(1, 2, 0), kernel,
            mode="constant", cval=0.0), (2, 0, 1))

    lst = np.nonzero(np.logical_and(msk, count > 0))
    fld[lst] = np.ma.nomask
    fld[lst] = nfld[lst] / count[lst]
    return fld


def date2day(date=default_epoch, epoch=default_epoch):
    """
    Compute the fractional number of days elapsed since the epoch to the date
    given.

    Parameters
    ----------
    date : datetime
        Input date
    epoch : datetime
        Date of epoch

    Returns
    -------
    numdays : list
    """
    date = np.atleast_1d(date)
    return [(t - epoch).total_seconds() * secs2day for t in date]


def day2date(day=0, epoch=default_epoch):
    """
    Return a datetime object from the number of days since the epoch<commit_msg>Add smoother function and change convolve_mask to use the smoother<commit_after>#!/usr/bin/env python
"""
  lib.py

  State Estimation and Analysis for PYthon

  Library of utilities for general seapy module, imported into the namespace
  when importing the seapy module

  Written by <NAME> on 10/18/13
  Copyright (c)2016 University of Hawaii under the BSD-License.
"""


import numpy as np
from scipy import ndimage
import os
import re
import datetime
import itertools


secs2day = 1.0 / 86400.0
default_epoch = datetime.datetime(2000, 1, 1)
_default_timeref = "days since " + default_epoch.strftime("%Y-%m-%m %H:%M:%S")


def adddim(fld, size=1):
    """
    replicate a field and add a new first dimension with given size

    Parameters
    ----------
    fld : array_like
        Input field.
    size : int, optional
        Size of additional first dimension

    Returns
    -------
    fld : array

    Examples
    --------
    >>> a=np.array([4, 5, 6, 7])
    >>> a.shape
    (4,)
    >>> b = seapy.adddim(a, 2)
    >>> b.shape
    (2, 4)
    >>> b
    array([[4, 5, 6, 7],
           [4, 5, 6, 7]])

    """
    fld = np.atleast_1d(fld)
    s = np.ones(fld.ndim + 1).astype(int)
    s[0] = int(size)
    return np.tile(fld, s)


def chunker(seq, size):
    """
    Iterate over an iterable in 'chunks' of a given size

    Parameters
    ----------
    seq : iterable,
        The sequence to iterate over
    size : int,
        The number of items to be returned in each 'chunk'

    Returns
    -------
    chunk : seq,
        The items of the chunk to be iterated

    Examples
    --------
    >>> x = [0,3,4,7,9,10,12,14]
    >>> for i in chunker(x, 3):
    >>>     print(i)
    [0, 3, 4]
    [7, 9, 10]
    [12, 14]

    """
    return (seq[pos:pos + size] for pos in range(0, len(seq), size))


def smooth(data, ksize=3, kernel=None, copy=True):
    """
    Smooth the data field using a specified convolution kernel
    or a default averaging kernel.

    Parameters
    ----------
    data : masked array_like
        Input field.
    ksize : int, optional
        Size of square kernel
    kernel : ndarray, optional
        Define a convolution kernel. Default is averaging
    copy : bool, optional
        If true, a copy of input array is made

    Returns
    -------
    fld : masked array
    """
    fld = np.ma.array(data, copy=copy)
    mask = np.ma.getmaskarray(fld).copy()

    # Make sure ksize is odd
    ksize = int(ksize + 1) if int(ksize) % 2 == 0 else int(ksize)
    if fld.ndim > 3 or fld.ndim < 2:
        raise AttributeError("Can only convolve 2- or 3-D fields")
    if ksize < 3:
        raise ValueError("ksize must be greater than or equal to 3")

    if kernel is None:
        kernel = np.ones((ksize, ksize)) / (ksize * ksize)
    else:
        ksize = kernel.shape[0]

    # First, convole over any masked values
    fld = convolve_mask(fld, ksize=ksize, copy=False)

    # Next, perform the convolution
    if fld.ndim == 2:
        fld = ndimage.convolve(fld.data, kernel,
                               mode="reflect", cval=0.0)
    else:
        kernel = np.expand_dims(kernel, axis=3)
        fld = np.transpose(ndimage.convolve(
            fld.filled(0).transpose(1, 2, 0), kernel,
            mode="reflect", cval=0.0), (2, 0, 1))

    # Apply the initial mask
    return np.ma.array(fld, mask=mask)


def smoother(data, ksize=3, kernel=None, copy=True, only_mask=False):
    """
    Convolve the kernel across the data to smooth or highlight
    the field.

    Parameters
    ----------
    data : masked array_like
        Input field.
    ksize : int, optional
        Size of square kernel
    kernel : ndarray, optional
        Define a convolution kernel. Default is averaging
    copy : bool, optional
        If true, a copy of input array is made
    only_mask : bool, optional
        If true, only consider the smoothing over the masked
        region

    Returns
    -------
    fld : masked array
    """
    fld = np.ma.array(data, copy=copy)
    if not copy:
        fld._sharedmask = False

    # Make sure ksize is odd
    ksize = int(ksize + 1) if int(ksize) % 2 == 0 else int(ksize)
    if fld.ndim > 3 or fld.ndim < 2:
        raise AttributeError("Can only convolve 2- or 3-D fields")
    if ksize < 3:
        raise ValueError("ksize must be greater than or equal to 3")

    if kernel is None:
        center = np.round(ksize / 2).astype(int)
        kernel = np.ones([ksize, ksize])
        kernel[center, center] = 0.0
    else:
        ksize = kernel.shape[0]

    # Convolve the mask
    msk = np.ma.getmaskarray(fld)
    if fld.ndim == 2:
        count = ndimage.convolve((~msk).view(np.int8), kernel,
                                 mode="constant", cval=0.0)
        nfld = ndimage.convolve(fld.data * (~msk).view(np.int8), kernel,
                                mode="constant", cval=0.0)
    else:
        kernel = np.expand_dims(kernel, axis=3)
        count = np.transpose(ndimage.convolve(
            (~msk).view(np.int8).transpose(1, 2, 0), kernel,
            mode="constant", cval=0.0), (2, 0, 1))
        nfld = np.transpose(ndimage.convolve(
            (fld.data * (~msk).view(np.int8)).transpose(1, 2, 0), kernel,
            mode="constant", cval=0.0), (2, 0, 1))

    if only_mask:
        lst = np.nonzero(np.logical_and(msk, count > 0))
        fld[lst] = np.ma.nomask
        fld[lst] = nfld[lst] / count[lst]
    else:
        lst = np.nonzero(~msk)
        fld[lst] = nfld[lst] / count[lst]
    return fld


def convolve_mask(data, ksize=3, kernel=None, copy=True):
    """
    Convolve data over the missing regions of a mask

    Parameters
    ----------
    data : masked array_like
        Input field.
    ksize : int, optional
        Size of square kernel
    kernel : ndarray, optional
        Define a convolution kernel. Default is averaging
    copy : bool, optional
        If true, a copy of input array is made

    Returns
    -------
    fld : masked array
    """
    return smoother(data, ksize, kernel, copy, True)


def date2day(date=default_epoch, epoch=default_epoch):
    """
    Compute the fractional number of days elapsed since the epoch to the date
    given.

    Parameters
    ----------
    date : datetime
        Input date
    epoch : datetime
        Date of epoch

    Returns
    -------
    numdays : list
    """
    date = np.atleast_1d(date)
    return [(t - epoch).total_seconds() * secs2day for t in date]


def day2date(day=0, epoch=default_epoch):
    """
    Return a datetime object from the number of days since the epoch
{'fraction_non_alphanumeric': 0.12661100753829943, 'fraction_numerical': 0.02261489827348626, 'mean_word_length': 2.0622983370563417, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 59, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '266775', 'n_tokens_mistral': 4320, 'n_tokens_neox': 4042, 'n_words': 1270}