<commit_before>
"""
Density assignation
-------------------
Module to assign geographically density value to a points.

TODO
----
- Use neighbourhood defintion?
- Recurrent measure (TODO)[better before with the population?]
"""

from scipy.spatial import KDTree
from scipy.spatial.distance import cdist
from scipy.stats import norm
from scipy.optimize import minimize
import numpy as np


def general_density_assignation(locs, retriever, info_ret, values, f_weights,
                                params_w, f_dens, params_d):
    """
    Parameters
    ----------
    locs: array_like shape(n, 2)
        location variables
    retriever: pySpatialTools.Retrieve.retrievers object
        retriever. Return the indices and distances of the possible retrivable
        points.
    values: array_like, shape (num. of retrievable candidates)
        values we will use to compute density.
    f_weighs: functions, str
        function of weighs assignation. It transforms the distance to weights.
    params_w: dict
        parameters needed to apply f_weighs.
    f_dens: function, set_scale_surgauss
        function of density assignation.
    params_d: dict
        parameters needed to apply f_dens.

    Returns
    -------
    M: array_like, shape(n)
        mesasure of each location given.

    """

    ## 0. Preparation needed variables
    #parameters = preparation_parameters(parameters)
    if len(values.shape) == 1:
        values = values.reshape(values.shape[0], 1)

    ## 1. Computation of density
    M = compute_measure(locs, retriever, info_ret, values, f_weights, params_w,
                        f_dens, params_d)

    return M


###############################################################################
############################### Compute measure ###############################
###############################################################################
def compute_measure(locs, retriever, info_ret, values, f_weighs, params_w,
                    f_dens, params_d):
    ## Computation of the measure based in the distances as weights.
    #M = np.zeros(locs.shape[0])
    M = []
    for i in xrange(locs.shape[0]):
        # Retrieve neighs_info
        neighs_info = retriever.retrieve_neighs(locs[[i]], info_ret[i], True)
        # Format neighs and dists
        neighs, dist, _, _ = neighs_info.get_information(k=0)
        neighs, dist = neighs[0][0], dist[0][0]
        neighs, dist = np.array(neighs).astype(int).ravel(), np.array(dist)
        # Get weights
        weights = from_distance_to_weights(dist, f_weighs, params_w)
        # Compute measure for i
        M_aux = compute_measure_i(weights, values[neighs], f_dens, params_d)
        M.append(M_aux)
        M = [0]
    M = np.array(M)
    return M


def compute_measure_i(weights, values, f_dens, params_d):
    "Swither function between different possible options to compute density."
    if type(f_dens) == str:
        if f_dens == 'weighted_count':
            measure = compute_measure_wcount(weights, values, **params_d)
        elif f_dens == 'weighted_avg':
            measure = compute_measure_wavg(weights, values, **params_d)
    else:
        measure = f_dens(weights, values, **params_d)

    return measure


def compute_measure_wcount(weights, values):
    """Measure to compute density only based on the weighted count of selected
    elements around the point considered.
    """
    measure = np.sum(weights)
    return measure


def compute_measure_wavg(weights, values):
    """Measure to compute density based on the weighted average of selected
    elements around the point considered.
    """
    measure = np.sum((weights * values.T).T, axis=0)
    return measure


# method, params (weitghted count, ...)
# method, params (linear, trapezoid,...)
###############################################################################
############################# Distance to weights #############################
###############################################################################
def from_distance_to_weights(dist, method, params):
    "Function which transforms the distance given to weights."

    if type(method) == str:
        if method == 'linear':
            weights = dist2weights_linear(dist, **params)
        elif method == 'Trapezoid':
            weights = dist2weights_trapez(dist, **params)
        elif method == 'inverse_prop':
            weights = dist2weights_invers(dist, **params)
        elif method == 'exponential':
            weights = dist2weights_exp(dist, **params)
        elif method == 'gaussian':
            weights = dist2weights_gauss(dist, **params)
        elif method == 'surgaussian':
            weights = dist2weights_surgauss(dist, **params)
        elif method == 'sigmoid':
            weights = dist2weights_sigmoid(dist, **params)
        else:
            weights = dist
    else:
        weights = method(dist, **params)
    weights = weights.ravel()
    return weights

<commit_msg>Interpolations minor changes.<commit_after>
"""
Density assignation
-------------------
Module to assign geographically density value to a points.

TODO
----
- Use neighbourhood defintion?
- Recurrent measure (TODO)[better before with the population?]
"""

from scipy.spatial import KDTree
from scipy.spatial.distance import cdist
from scipy.stats import norm
from scipy.optimize import minimize
import numpy as np


def general_density_assignation(locs, retriever, info_ret, values, f_weights,
                                params_w, f_dens, params_d):
    """
    Parameters
    ----------
    locs: array_like shape(n, 2)
        location variables
    retriever: pySpatialTools.Retrieve.retrievers object
        retriever. Return the indices and distances of the possible retrivable
        points.
    values: array_like, shape (num. of retrievable candidates)
        values we will use to compute density.
    f_weighs: functions, str
        function of weighs assignation. It transforms the distance to weights.
    params_w: dict
        parameters needed to apply f_weighs.
    f_dens: function, set_scale_surgauss
        function of density assignation.
    params_d: dict
        parameters needed to apply f_dens.

    Returns
    -------
    M: array_like, shape(n)
        mesasure of each location given.

    """

    ## 0. Preparation needed variables
    #parameters = preparation_parameters(parameters)
    if len(values.shape) == 1:
        values = values.reshape(values.shape[0], 1)

    ## 1. Computation of density
    M = compute_measure(locs, retriever, info_ret, values, f_weights, params_w,
                        f_dens, params_d)

    return M


###############################################################################
############################### Compute measure ###############################
###############################################################################
def compute_measure(locs, retriever, info_ret, values, f_weighs, params_w,
                    f_dens, params_d):
    ## Computation of the measure based in the distances as weights.
    #M = np.zeros(locs.shape[0])
    M = []
    for i in xrange(locs.shape[0]):
        # Retrieve neighs_info
        neighs_info = retriever.retrieve_neighs(locs[[i]], info_ret[i], True)
        # Format neighs and dists
        neighs, dist, _, _ = neighs_info.get_information(k=0)
        neighs, dist = neighs[0][0], dist[0][0]
        neighs, dist = np.array(neighs).astype(int).ravel(), np.array(dist)
        # Get weights
        weights = from_distance_to_weights(dist, f_weighs, params_w)
        # Compute measure for i
        M_aux = compute_measure_i(weights, values[neighs], f_dens, params_d)
        M.append(M_aux)
        M = [0]
    M = np.array(M)
    return M


def compute_measure_i(weights, values, f_dens, params_d):
    "Swither function between different possible options to compute density."
    if type(f_dens) == str:
        if f_dens == 'weighted_count':
            measure = compute_measure_wcount(weights, values, **params_d)
        elif f_dens == 'weighted_avg':
            measure = compute_measure_wavg(weights, values, **params_d)
        elif f_dens == 'null':
            measure = compute_measure_null(weights, values, **params_d)
    else:
        measure = f_dens(weights, values, **params_d)

    return measure


def compute_measure_wcount(weights, values):
    """Measure to compute density only based on the weighted count of selected
    elements around the point considered.
    """
    measure = np.sum(weights)
    return measure


def compute_measure_wavg(weights, values):
    """Measure to compute density based on the weighted average of selected
    elements around the point considered.
    """
    measure = np.sum((weights * values.T).T, axis=0)
    return measure


def compute_measure_null(weights, values):
    measure = values[0]
    return measure


# method, params (weitghted count, ...)
# method, params (linear, trapezoid,...)
###############################################################################
############################# Distance to weights #############################
###############################################################################
def from_distance_to_weights(dist, method, params):
    "Function which transforms the distance given to weights."

    if type(method) == str:
        if method == 'linear':
            weights = dist2weights_linear(dist, **params)
        elif method == 'Trapezoid':
            weights = dist2weights_trapez(dist, **params)
        elif method == 'inverse_prop':
            weights = dist2weights_invers(dist, **params)
        elif method == 'exponential':
            weights = dist2weights_exp(dist, **params)
        elif method == 'gaussian':
            weights = dist2weights_gauss(dist, **params)
        elif method == 'surgaussian':
            weights = dist2weights_surgauss(dist, **params)
        elif method == 'sigmoid':
            weights = dist2weights_sigmoid(dist, **params)
        else:
            weights = dist
    else:
        weights = method(dist, **params)
    weights = np.array(weights).ravel()
    return weights


{'fraction_non_alphanumeric': 0.17641277641277642, 'fraction_numerical': 0.004422604422604423, 'mean_word_length': 3.050955414012739, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '19236', 'n_tokens_mistral': 2897, 'n_tokens_neox': 2751, 'n_words': 804}