<commit_before># ---------------------------------------------------------------#
#   astroNN.nn.numpy: tools written with numpy instead of tf
# ---------------------------------------------------------------#
import astropy.units as u
import numpy as np
from astroNN.config import MAGIC_NUMBER


def sigmoid(x):
    """
    NAME: sigmoid
    PURPOSE: numpy implementation of tf.sigmoid
    INPUT:
        x (ndarray): input
    OUTPUT:
        (ndarray)
    HISTORY:
        2018-Apr-11 - Written - <NAME> (University of Toronto)
    """
    return 1 / (1 + np.exp(-x))


def sigmoid_inv(x):
    """
    NAME: sigmoid_inv
    PURPOSE: numpy implementation of tf.sigmoid inverse
    INPUT:
        x (ndarray): input
    OUTPUT:
        (ndarray)
    HISTORY:
        2018-Apr-11 - Written - <NAME> (University of Toronto)
    """
    return np.log(x / (1 - x))


def l1(x, l1=0.):
    """
    NAME: l1
    PURPOSE: numpy implementation of tf.keras.regularizers.l1
    INPUT:
        x (ndarray): input
    OUTPUT:
        (ndarray)
    HISTORY:
        2018-Apr-11 - Written - <NAME> (University of Toronto)
    """
    l1_x = 0.
    l1_x += np.sum(l1 * np.abs(x))
    return l1_x


def l2(x, l2=0.):
    """
    NAME: l2
    PURPOSE: numpy implementation of tf.keras.regularizers.l2
    INPUT:
        x (ndarray): input
    OUTPUT:
        (ndarray) representing regularising term
    HISTORY:
        2018-Apr-11 - Written - <NAME> (University of Toronto)
    """
    l2_x = 0.
    l2_x += np.sum(l2 * np.square(x))
    return l2_x


def relu(x):
    """
    NAME: relu
    PURPOSE: numpy implementation of tf.nn.relu
    INPUT:
        x (ndarray): input
    OUTPUT:
        (ndarray) representing activated ndarray
    HISTORY:
        2018-Apr-11 - Written - <NAME> (University of Toronto)
    """
    return x * (x > 0)


def mean_absolute_percentage_error(x, y, axis=None):
    """
    NAME: mean_absolute_percentage_error
    PURPOSE:
        mean_absolute_percentage_error using numpy abs(x-y)/y
        preserve magic_number
    INPUT:
        x (ndarray, astropy quantity): prediction
        y (ndarray, astropy quantity): ground truth
        axis (int): numpy axis
    OUTPUT:
        (ndarray) representing activated ndarray
    HISTORY:
        2018-Apr-11 - Written - <NAME> (University of Toronto)
    """
    if type(x) == u.quantity.Quantity:
        x = x.value
    if type(y) == u.quantity.Quantity:
        y = y.value

    return np.ma.array(np.abs((x-y)/y)*100., mask=[(x == MAGIC_NUMBER) | (y == MAGIC_NUMBER)]).mean(axis=axis)


def mean_absolute_error(x, y, axis=None):
    """
    NAME: mean_absolute_error
    PURPOSE:
        mean_absolute_error using numpy abs(x-y)/y
        preserve magic_number
    INPUT:
        x (ndarray, astropy quantity): prediction
        y (ndarray, astropy quantity): ground truth
        axis (int): numpy axis
    OUTPUT:
        (ndarray) representing activated ndarray
    HISTORY:
        2018-Apr-11 - Written - <NAME> (University of Toronto)
    """
    if type(x) == u.quantity.Quantity:
        x = x.value
    if type(y) == u.quantity.Quantity:
        y = y.value

    return np.ma.array(np.abs(x-y), mask=[(x == MAGIC_NUMBER) | (y == MAGIC_NUMBER)]).mean(axis=axis)
<commit_msg>bettwe deal with astropy unit in numpy tools<commit_after># ---------------------------------------------------------------#
#   astroNN.nn.numpy: tools written with numpy instead of tf
# ---------------------------------------------------------------#
import astropy.units as u
import numpy as np
from astroNN.config import MAGIC_NUMBER


def sigmoid(x):
    """
    NAME: sigmoid
    PURPOSE: numpy implementation of tf.sigmoid
    INPUT:
        x (ndarray): input
    OUTPUT:
        (ndarray)
    HISTORY:
        2018-Apr-11 - Written - <NAME> (University of Toronto)
    """
    return 1 / (1 + np.exp(-x))


def sigmoid_inv(x):
    """
    NAME: sigmoid_inv
    PURPOSE: numpy implementation of tf.sigmoid inverse
    INPUT:
        x (ndarray): input
    OUTPUT:
        (ndarray)
    HISTORY:
        2018-Apr-11 - Written - <NAME> (University of Toronto)
    """
    return np.log(x / (1 - x))


def l1(x, l1=0.):
    """
    NAME: l1
    PURPOSE: numpy implementation of tf.keras.regularizers.l1
    INPUT:
        x (ndarray): input
    OUTPUT:
        (ndarray)
    HISTORY:
        2018-Apr-11 - Written - <NAME> (University of Toronto)
    """
    l1_x = 0.
    l1_x += np.sum(l1 * np.abs(x))
    return l1_x


def l2(x, l2=0.):
    """
    NAME: l2
    PURPOSE: numpy implementation of tf.keras.regularizers.l2
    INPUT:
        x (ndarray): input
    OUTPUT:
        (ndarray) representing regularising term
    HISTORY:
        2018-Apr-11 - Written - <NAME> (University of Toronto)
    """
    l2_x = 0.
    l2_x += np.sum(l2 * np.square(x))
    return l2_x


def relu(x):
    """
    NAME: relu
    PURPOSE: numpy implementation of tf.nn.relu
    INPUT:
        x (ndarray): input
    OUTPUT:
        (ndarray) representing activated ndarray
    HISTORY:
        2018-Apr-11 - Written - <NAME> (University of Toronto)
    """
    return x * (x > 0)


def mean_absolute_percentage_error(x, y, axis=None):
    """
    NAME: mean_absolute_percentage_error
    PURPOSE:
        mean_absolute_percentage_error using numpy abs(x-y)/y
        preserve magic_number
    INPUT:
        x (ndarray, astropy quantity): prediction
        y (ndarray, astropy quantity): ground truth
        axis (int): numpy axis
    OUTPUT:
        (ndarray) representing activated ndarray
    HISTORY:
        2018-Apr-11 - Written - <NAME> (University of Toronto)
    """
    if type(x) == u.quantity.Quantity and type(y) == u.quantity.Quantity:
        percetnage = ((x - y) / y).value
        # still need to take the value for creating mask
        x = x.value
        y = y.value
    else:
        percetnage = (x - y) / y
    return np.ma.array(np.abs(percetnage) * 100., mask=[(x == MAGIC_NUMBER) | (y == MAGIC_NUMBER)]).mean(axis=axis)


def mean_absolute_error(x, y, axis=None):
    """
    NAME: mean_absolute_error
    PURPOSE:
        mean_absolute_error using numpy abs(x-y)
        preserve magic_number
    INPUT:
        x (ndarray, astropy quantity): prediction
        y (ndarray, astropy quantity): ground truth
        axis (int): numpy axis
    OUTPUT:
        (ndarray) representing activated ndarray
    HISTORY:
        2018-Apr-11 - Written - <NAME> (University of Toronto)
    """
    if type(x) == u.quantity.Quantity and type(y) == u.quantity.Quantity:
        diff = (x - y).value
        # still need to take the value for creating mask
        x = x.value
        y = y.value
    else:
        diff = (x - y)
    return np.ma.array(np.abs(diff), mask=[(x == MAGIC_NUMBER) | (y == MAGIC_NUMBER)]).mean(axis=axis)

{'fraction_non_alphanumeric': 0.15162667451788606, 'fraction_numerical': 0.02031503017812454, 'mean_word_length': 1.9824407374890254, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5352512', 'n_tokens_mistral': 2362, 'n_tokens_neox': 2209, 'n_words': 602}