<commit_before># -*- coding: utf-8 -*-

import sys
import numpy as np
from scipy.stats import norm
from scipy.integrate import quad

import time as tm  # For testing

from .utils import P_to_A, A_to_P
from . import constants as c
# from . import darts
from . import priors


def ln_posterior(x, dart):
    """
    Calculate the natural log of the posterior probability. This function
    calls `ln_prior` to calculate the prior probability of model parameters and
    then `ln_likelihood` to evaluate the likelihood.

    Args:
        x : tuple, set of model parameters
        dart : DartBoard instance, positions of the dart

    Returns:
        lp : float, natural log of the posterior probability
    """

    # Empty array for emcee blobs
    empty_arr = np.zeros(17)


    # Calculate the prior probability
    lp = priors.ln_prior(x, dart)

    if dart.ntemps is None:

        if np.isinf(lp) or np.isnan(lp): return -np.inf, empty_arr

        ll, output = ln_likelihood(x, dart)

        # Convert from numpy structured array to a regular ndarray
        if len(output.shape) == 0:
            output = np.column_stack(output[name] for name in output.dtype.names)[0]


        return lp+ll, np.array([output])

    else:

        if np.isinf(lp) or np.isnan(lp): return -np.inf

        ll = ln_likelihood(x, dart)
        return lp+ll


def ln_likelihood(x, dart):
    """
    Calculate the natural log of the likelihood. This function first calls the
    binary evolution code given the set of model parameters, then checks to make
    sure the resulting binary is of the type specified by the user. Then, it
    calls the `posterior_properties` function to account for observations.

    Args:
        x : tuple, set of model parameters
        dart : DartBoard instance

    Returns:
        lp : float, natural log of the likelihood
    """

    # For use later in posterior properties
    x_in = x

    # Save model parameters to variables
    ln_M1, ln_M2, ln_a, ecc = x[0:4]
    x = x[4:]
    if dart.first_SN:
        v_kick1, theta_kick1, phi_kick1 = x[0:3]
        x = x[3:]
    else:
        v_kick1, theta_kick1, phi_kick1 = 0.0, 0.0, 0.0
    if dart.second_SN:
        v_kick2, theta_kick2, phi_kick2 = x[0:3]
        x = x[3:]
    if dart.prior_pos is not None:
        ra_b, dec_b = x[0:2]
        x = x[2:]
    if dart.model_metallicity:
        ln_z = x[0]
        z = np.exp(ln_z)
        x = x[1:]
    else:
        z = dart.metallicity
    ln_t_b = x[0]


    # Move from log vars to linear
    M1 = np.exp(ln_M1)
    M2 = np.exp(ln_M2)
    a = np.exp(ln_a)
    t_b = np.exp(ln_t_b)


    # Empty array for emcee blobs
    empty_arr = np.zeros(17)


    # Get initial orbital period
    orbital_period = A_to_P(M1, M2, a)


    # Proxy values if binary_type does not include second SN
    if not dart.second_SN:
        v_kick2 = v_kick1
        theta_kick2 = theta_kick1
        phi_kick2 = phi_kick1


    # Run rapid binary evolution code
    output = dart.evolve_binary(M1, M2, orbital_period, ecc,
                                v_kick1, theta_kick1, phi_kick1,
                                v_kick2, theta_kick2, phi_kick2,
                                t_b, z, False, **dart.model_kwargs)

    # Return posterior probability and blobs
    if not check_output(output, dart.binary_type):
        if dart.ntemps is None:
            return -np.inf, empty_arr
        else:
            return -np.inf


    # Check for kwargs arguments
    ll = 0
    if not dart.system_kwargs == {}: ll = posterior_properties(x_in, output, dart)

    if dart.ntemps is None:
        return ll, output
    else:
        return ll




def posterior_properties(x, output, dart):
    """
    Calculate the (log of the) likelihood given specific observables.
    This function evaluates the likelihood of the model parameters. This function
    calculates the likelihood for the model parameters to account for the
    observational constraints provided.

    Args:
        x : tuple, set of model parameters
        output : ndarray, binary evolution output
        dart : DartBoard class, provides the necessary functions

    Returns:
        likelihood : Likelihood of the set of model parameters.

    """

    P_orb_out = A_to_P(output['M1'], output['M2'], output['a'])


    # Save model parameters to variables
    ln_M1, ln_M2, ln_a, ecc = x[0:4]
    x = x[4:]
    if dart.first_SN:
        v_kick1, theta_kick1, phi_kick1 = x[0:3]
        x = x[3:]
    if dart.second_SN:
        v_kick2, theta_kick2, phi_kick2 = x[0:3]
        x = x[3:]
    if dart.prior_pos is not None:
        ra_b, dec_b = x[0:2]
        x = x[2:]
    if dart.model_metallicity:
        ln_z = x[0]
        z = np.exp(ln_z)
        x = x[1:]
    else:
        z = dart.metallicity
    ln_t_b = x[0]


    M1 = np.exp(ln_M1)
    M2 = np.exp(ln_M2)
    a = np.exp(ln_a)
    t_b = np.exp(ln_t_b)

    # Calculate an X-ray luminosity
    L_x_out = calculate_L_x(output['M1'], output['mdot1'], output['k1'])


    def get_error_from_kwargs(param, **kwargs):

        # Search kwargs for parameter uncertainty
        for key, error in kwargs.items():
            if key == param + "_err":
                return error

        # Parameter uncertainty was not provided
        print("You must provide an uncertainty for the observables. Example:")
        print("M2 : 8.0, M2_err : 1.0")
        sys.exit(-1)


    # Possible observables
    observables = ["M1", "M2", "M_tot", "P_orb", "a", "ecc", "L_x", "v_sys",
                   "R1", "R2", "Teff1", "Teff2", "L1", "L2",'Mdot1', 'Mdot2']
    model_vals = [output['M1'], output['M2'], output['M1']+output['M2'],
                  P_orb_out, output['a'], output['ecc'], L_x_out,
                  output['v_sys'], output['R1'], output['R2'], output['Teff1'],
                  output['Teff2'], output['L1'], output['L2'], output['mdot1'],
                  output['mdot2'] ]


    # Add log probabilities for each observable
    ll = 0
    for key, value in dart.system_kwargs.items():
        for i,param in enumerate(observables):
            if key == param:
                error = get_error_from_kwargs(param, **dart.system_kwargs)
                likelihood = norm.pdf(model_vals[i], loc=value, scale=error)
                if likelihood == 0.0: return -np.inf
                ll += np.log(likelihood)

        # Constraints on k-type
        if key == 'k1':
            if not output['k1'] in value: return -np.inf
        if key == 'k2':
            if not output['k2'] in value: return -np.inf

        # Mass function must be treated specially
        if key == "m_f_1":
            error = get_error_from_kwargs("m_f_1", **dart.system_kwargs)
            likelihood = calc_prob_from_mass_function(output['M1'], output['M2'], value, error)
            if likelihood == 0.0: return -np.inf
            ll += np.log(likelihood)

        if key == "m_f_2":
            error = get_error_from_kwargs("m_f_2", **dart.system_kwargs)
            likelihood = calc_prob_from_mass_function(output['M2'], output['M1'], value, error)
            if likelihood == 0.0: return -np.inf
            ll += np.log(likelihood)

        if key == 'L_x_min':
            if L_x_out < value: return -np.inf
        if key == 'L_x_max':
            if L_x_out > value: return -np.inf

        if key == "ecc_max":
            if output['ecc'] > value: return -np.inf

        # Age of the compact object, based on e.g., modeling the SN remnant
        # as in e.g., Circ X-1
        if key == 'NS_age_max':
            if t_b - output['t_SN1'] > value: return -np.inf


    # Add log probabilities if position is provided
    if dart.ra_obs is not None and dart.dec_obs is not None:

        # Projected travel angle
        theta_proj = get_theta_proj(c.deg_to_rad*dart.ra_obs, c.deg_to_rad*dart.dec_obs, \
                                    c.deg_to_rad*ra_b, c.deg_to_rad*dec_b)

        # Travel time in seconds
        t_sn = (t_b - output['t_SN1']) * 1.0e6 * c.yr_to_sec

        # Maximum angle in radians
        angle_max = (output['v_sys'] * t_sn) / c.distance

        # Define conditional
        conds = [theta_proj>=angle_max, theta_proj<angle_max]
        funcs = [lambda theta_proj: -np.inf,
                 lambda theta_proj: np.log(np.tan(np.arcsin(theta_proj/angle_max))/angle_max)]
                #  lambda theta_proj: np.log( theta_proj / angle_max**2 / np.sqrt(1.0 - (theta_proj/angle_max)**2) )]

        # Jacobian for coordinate change - ra, dec in radians
        J_coor = np.abs(get_J_coor(c.deg_to_rad*dart.ra_obs, c.deg_to_rad*dart.dec_obs, \
                        c.deg_to_rad*ra_b, c.deg_to_rad*dec_b))
        J_coor *= c.deg_to_rad * c.deg_to_rad  # Change jacobian from radians to degrees

        P_omega = 1.0 / (2.0 * np.pi)

        # Likelihood
        ll += np.piecewise(theta_proj, conds, funcs) + np.log(P_omega) + np.log(J_coor)

    return ll



def calculate_ADAF_efficiency(Mdot, Mdot_edd, delta=0.1):
    """
    Calculate the mass-luminosity efficiency based on the ADAF
    model of Xie & Yuan (2012), using the piecewise fitting function
    using values provided their Table 1 for their Equation 11.

    Args:
        Mdot : mass transfer rate onto a BH
        Mdot_edd : Eddington mass transfer rate (same units as Mdot)
        delta : (optional) fraction of ionization energy acting on electrons

    Returns:
        epsilon : mass-light conversion efficiency
    """


    M_ratio = Mdot/Mdot_edd

    if not delta in [0.5, 0.1, 0.01, 0.001]:
        print("You must provide an acceptable value for delta.")
        sys.exit(-1)

    if delta == 0.5:
        conds = [M_ratio < 2.9e-5,
                 (M_ratio >= 2.9e-5) & (M_ratio < 3.3e-3),
                 (M_ratio >= 3.3e-3) & (M_ratio < 5.3e-3),
                 M_ratio >= 5.3e-3]
        func_epsilon = [lambda M_ratio: 1.58*(100*M_ratio)**0.65,
                        lambda M_ratio: 0.055*(100*M_ratio)**0.076,
                        lambda M_ratio: 0.17*(100*M_ratio)**1.12,
                        lambda M_ratio: 0.1]

    if delta == 0.1:
        conds = [M_ratio < 9.4e-5,
                 (M_ratio >= 9.4e-5) & (M_ratio < 5.0e-3),
                 (M_ratio >= 5.0e-3) & (M_ratio < 6.6e-3),
                 M_ratio >= 6.6e-3]
        func_epsilon = [lambda M_ratio: 0.12*(100*M_ratio)**0.59,
                        lambda M_ratio: 0.026*(100*M_ratio)**0.27,
                        lambda M_ratio: 0.50*(100*M_ratio)**4.53,
                        lambda M_ratio: 0.1]

    if delta == 1.0e-2:
        conds = [M_ratio < 1.6e-5,
                 (M_ratio >= 1.6e-5) & (M_ratio < 5.3e-3),
                 (M_ratio >= 5.3e-3) & (M_ratio < 7.1e-3),
                 M_ratio >= 7.1e-3]
        func_epsilon = [lambda M_ratio: 0.069*(100*M_ratio)**0.69,
                        lambda M_ratio: 0.027*(100*M_ratio)**0.54,
                        lambda M_ratio: 0.42*(100*M_ratio)**4.85,
                        lambda M_ratio: 0.1]

    if delta == 1.0e-3:
        conds = [M_ratio < 7.6e-5,
                 (M_ratio >= 7.6e-5) & (M_ratio < 4.5e-3),
                 (M_ratio >= 4.5e-3) & (M_ratio < 7.1e-3),
                 M_ratio >= 7.1e-3]
        func_epsilon = [lambda M_ratio: 0.065*(100*M_ratio)**0.71,
                        lambda M_ratio: 0.020*(100*M_ratio)**0.47,
                        lambda M_ratio: 0.26*(100*M_ratio)**3.67,
                        lambda M_ratio: 0.1]

    # For compatibility with older version of np.piecewise, ensure conds is an ndarray
    if np.isscalar(conds) or \
           not (isinstance(conds[0], list) or
                isinstance(conds[0], np.ndarray)):
        conds = [conds]
    conds = [asarray(c, dtype=bool) for c in conds]

    return np.piecewise(M_ratio, conds, func_epsilon)
<commit_msg>Fixed another stupid buggit add dart_board/posterior.py<commit_after># -*- coding: utf-8 -*-

import sys
import numpy as np
from scipy.stats import norm
from scipy.integrate import quad

import time as tm  # For testing

from .utils import P_to_A, A_to_P
from . import constants as c
# from . import darts
from . import priors


def ln_posterior(x, dart):
    """
    Calculate the natural log of the posterior probability. This function
    calls `ln_prior` to calculate the prior probability of model parameters and
    then `ln_likelihood` to evaluate the likelihood.

    Args:
        x : tuple, set of model parameters
        dart : DartBoard instance, positions of the dart

    Returns:
        lp : float, natural log of the posterior probability
    """

    # Empty array for emcee blobs
    empty_arr = np.zeros(17)


    # Calculate the prior probability
    lp = priors.ln_prior(x, dart)

    if dart.ntemps is None:

        if np.isinf(lp) or np.isnan(lp): return -np.inf, empty_arr

        ll, output = ln_likelihood(x, dart)

        # Convert from numpy structured array to a regular ndarray
        if len(output.shape) == 0:
            output = np.column_stack(output[name] for name in output.dtype.names)[0]


        return lp+ll, np.array([output])

    else:

        if np.isinf(lp) or np.isnan(lp): return -np.inf

        ll = ln_likelihood(x, dart)
        return lp+ll


def ln_likelihood(x, dart):
    """
    Calculate the natural log of the likelihood. This function first calls the
    binary evolution code given the set of model parameters, then checks to make
    sure the resulting binary is of the type specified by the user. Then, it
    calls the `posterior_properties` function to account for observations.

    Args:
        x : tuple, set of model parameters
        dart : DartBoard instance

    Returns:
        lp : float, natural log of the likelihood
    """

    # For use later in posterior properties
    x_in = x

    # Save model parameters to variables
    ln_M1, ln_M2, ln_a, ecc = x[0:4]
    x = x[4:]
    if dart.first_SN:
        v_kick1, theta_kick1, phi_kick1 = x[0:3]
        x = x[3:]
    else:
        v_kick1, theta_kick1, phi_kick1 = 0.0, 0.0, 0.0
    if dart.second_SN:
        v_kick2, theta_kick2, phi_kick2 = x[0:3]
        x = x[3:]
    if dart.prior_pos is not None:
        ra_b, dec_b = x[0:2]
        x = x[2:]
    if dart.model_metallicity:
        ln_z = x[0]
        z = np.exp(ln_z)
        x = x[1:]
    else:
        z = dart.metallicity
    ln_t_b = x[0]


    # Move from log vars to linear
    M1 = np.exp(ln_M1)
    M2 = np.exp(ln_M2)
    a = np.exp(ln_a)
    t_b = np.exp(ln_t_b)


    # Empty array for emcee blobs
    empty_arr = np.zeros(17)


    # Get initial orbital period
    orbital_period = A_to_P(M1, M2, a)


    # Proxy values if binary_type does not include second SN
    if not dart.second_SN:
        v_kick2 = v_kick1
        theta_kick2 = theta_kick1
        phi_kick2 = phi_kick1


    # Run rapid binary evolution code
    output = dart.evolve_binary(M1, M2, orbital_period, ecc,
                                v_kick1, theta_kick1, phi_kick1,
                                v_kick2, theta_kick2, phi_kick2,
                                t_b, z, False, **dart.model_kwargs)

    # Return posterior probability and blobs
    if not check_output(output, dart.binary_type):
        if dart.ntemps is None:
            return -np.inf, empty_arr
        else:
            return -np.inf


    # Check for kwargs arguments
    ll = 0
    if not dart.system_kwargs == {}: ll = posterior_properties(x_in, output, dart)

    if dart.ntemps is None:
        return ll, output
    else:
        return ll




def posterior_properties(x, output, dart):
    """
    Calculate the (log of the) likelihood given specific observables.
    This function evaluates the likelihood of the model parameters. This function
    calculates the likelihood for the model parameters to account for the
    observational constraints provided.

    Args:
        x : tuple, set of model parameters
        output : ndarray, binary evolution output
        dart : DartBoard class, provides the necessary functions

    Returns:
        likelihood : Likelihood of the set of model parameters.

    """

    P_orb_out = A_to_P(output['M1'], output['M2'], output['a'])


    # Save model parameters to variables
    ln_M1, ln_M2, ln_a, ecc = x[0:4]
    x = x[4:]
    if dart.first_SN:
        v_kick1, theta_kick1, phi_kick1 = x[0:3]
        x = x[3:]
    if dart.second_SN:
        v_kick2, theta_kick2, phi_kick2 = x[0:3]
        x = x[3:]
    if dart.prior_pos is not None:
        ra_b, dec_b = x[0:2]
        x = x[2:]
    if dart.model_metallicity:
        ln_z = x[0]
        z = np.exp(ln_z)
        x = x[1:]
    else:
        z = dart.metallicity
    ln_t_b = x[0]


    M1 = np.exp(ln_M1)
    M2 = np.exp(ln_M2)
    a = np.exp(ln_a)
    t_b = np.exp(ln_t_b)

    # Calculate an X-ray luminosity
    L_x_out = calculate_L_x(output['M1'], output['mdot1'], output['k1'])


    def get_error_from_kwargs(param, **kwargs):

        # Search kwargs for parameter uncertainty
        for key, error in kwargs.items():
            if key == param + "_err":
                return error

        # Parameter uncertainty was not provided
        print("You must provide an uncertainty for the observables. Example:")
        print("M2 : 8.0, M2_err : 1.0")
        sys.exit(-1)


    # Possible observables
    observables = ["M1", "M2", "M_tot", "P_orb", "a", "ecc", "L_x", "v_sys",
                   "R1", "R2", "Teff1", "Teff2", "L1", "L2",'Mdot1', 'Mdot2']
    model_vals = [output['M1'], output['M2'], output['M1']+output['M2'],
                  P_orb_out, output['a'], output['ecc'], L_x_out,
                  output['v_sys'], output['R1'], output['R2'], output['Teff1'],
                  output['Teff2'], output['L1'], output['L2'], output['mdot1'],
                  output['mdot2'] ]


    # Add log probabilities for each observable
    ll = 0
    for key, value in dart.system_kwargs.items():
        for i,param in enumerate(observables):
            if key == param:
                error = get_error_from_kwargs(param, **dart.system_kwargs)
                likelihood = norm.pdf(model_vals[i], loc=value, scale=error)
                if likelihood == 0.0: return -np.inf
                ll += np.log(likelihood)

        # Constraints on k-type
        if key == 'k1':
            if not output['k1'] in value: return -np.inf
        if key == 'k2':
            if not output['k2'] in value: return -np.inf

        # Mass function must be treated specially
        if key == "m_f_1":
            error = get_error_from_kwargs("m_f_1", **dart.system_kwargs)
            likelihood = calc_prob_from_mass_function(output['M1'], output['M2'], value, error)
            if likelihood == 0.0: return -np.inf
            ll += np.log(likelihood)

        if key == "m_f_2":
            error = get_error_from_kwargs("m_f_2", **dart.system_kwargs)
            likelihood = calc_prob_from_mass_function(output['M2'], output['M1'], value, error)
            if likelihood == 0.0: return -np.inf
            ll += np.log(likelihood)

        if key == 'L_x_min':
            if L_x_out < value: return -np.inf
        if key == 'L_x_max':
            if L_x_out > value: return -np.inf

        if key == "ecc_max":
            if output['ecc'] > value: return -np.inf

        # Age of the compact object, based on e.g., modeling the SN remnant
        # as in e.g., Circ X-1
        if key == 'NS_age_max':
            if t_b - output['t_SN1'] > value: return -np.inf


    # Add log probabilities if position is provided
    if dart.ra_obs is not None and dart.dec_obs is not None:

        # Projected travel angle
        theta_proj = get_theta_proj(c.deg_to_rad*dart.ra_obs, c.deg_to_rad*dart.dec_obs, \
                                    c.deg_to_rad*ra_b, c.deg_to_rad*dec_b)

        # Travel time in seconds
        t_sn = (t_b - output['t_SN1']) * 1.0e6 * c.yr_to_sec

        # Maximum angle in radians
        angle_max = (output['v_sys'] * t_sn) / c.distance

        # Define conditional
        conds = [theta_proj>=angle_max, theta_proj<angle_max]
        funcs = [lambda theta_proj: -np.inf,
                 lambda theta_proj: np.log(np.tan(np.arcsin(theta_proj/angle_max))/angle_max)]
                #  lambda theta_proj: np.log( theta_proj / angle_max**2 / np.sqrt(1.0 - (theta_proj/angle_max)**2) )]

        # Jacobian for coordinate change - ra, dec in radians
        J_coor = np.abs(get_J_coor(c.deg_to_rad*dart.ra_obs, c.deg_to_rad*dart.dec_obs, \
                        c.deg_to_rad*ra_b, c.deg_to_rad*dec_b))
        J_coor *= c.deg_to_rad * c.deg_to_rad  # Change jacobian from radians to degrees

        P_omega = 1.0 / (2.0 * np.pi)

        # Likelihood
        ll += np.piecewise(theta_proj, conds, funcs) + np.log(P_omega) + np.log(J_coor)

    return ll



def calculate_ADAF_efficiency(Mdot, Mdot_edd, delta=0.1):
    """
    Calculate the mass-luminosity efficiency based on the ADAF
    model of Xie & Yuan (2012), using the piecewise fitting function
    using values provided their Table 1 for their Equation 11.

    Args:
        Mdot : mass transfer rate onto a BH
        Mdot_edd : Eddington mass transfer rate (same units as Mdot)
        delta : (optional) fraction of ionization energy acting on electrons

    Returns:
        epsilon : mass-light conversion efficiency
    """


    M_ratio = Mdot/Mdot_edd

    if not delta in [0.5, 0.1, 0.01, 0.001]:
        print("You must provide an acceptable value for delta.")
        sys.exit(-1)

    if delta == 0.5:
        conds = [M_ratio < 2.9e-5,
                 (M_ratio >= 2.9e-5) & (M_ratio < 3.3e-3),
                 (M_ratio >= 3.3e-3) & (M_ratio < 5.3e-3),
                 M_ratio >= 5.3e-3]
        func_epsilon = [lambda M_ratio: 1.58*(100*M_ratio)**0.65,
                        lambda M_ratio: 0.055*(100*M_ratio)**0.076,
                        lambda M_ratio: 0.17*(100*M_ratio)**1.12,
                        lambda M_ratio: 0.1]

    if delta == 0.1:
        conds = [M_ratio < 9.4e-5,
                 (M_ratio >= 9.4e-5) & (M_ratio < 5.0e-3),
                 (M_ratio >= 5.0e-3) & (M_ratio < 6.6e-3),
                 M_ratio >= 6.6e-3]
        func_epsilon = [lambda M_ratio: 0.12*(100*M_ratio)**0.59,
                        lambda M_ratio: 0.026*(100*M_ratio)**0.27,
                        lambda M_ratio: 0.50*(100*M_ratio)**4.53,
                        lambda M_ratio: 0.1]

    if delta == 1.0e-2:
        conds = [M_ratio < 1.6e-5,
                 (M_ratio >= 1.6e-5) & (M_ratio < 5.3e-3),
                 (M_ratio >= 5.3e-3) & (M_ratio < 7.1e-3),
                 M_ratio >= 7.1e-3]
        func_epsilon = [lambda M_ratio: 0.069*(100*M_ratio)**0.69,
                        lambda M_ratio: 0.027*(100*M_ratio)**0.54,
                        lambda M_ratio: 0.42*(100*M_ratio)**4.85,
                        lambda M_ratio: 0.1]

    if delta == 1.0e-3:
        conds = [M_ratio < 7.6e-5,
                 (M_ratio >= 7.6e-5) & (M_ratio < 4.5e-3),
                 (M_ratio >= 4.5e-3) & (M_ratio < 7.1e-3),
                 M_ratio >= 7.1e-3]
        func_epsilon = [lambda M_ratio: 0.065*(100*M_ratio)**0.71,
                        lambda M_ratio: 0.020*(100*M_ratio)**0.47,
                        lambda M_ratio: 0.26*(100*M_ratio)**3.67,
                        lambda M_ratio: 0.1]

    # For compatibility with older version of np.piecewise, ensure conds is an ndarray
    if np.isscalar(conds) or \
           not (isinstance(conds[0], list) or
                isinstance(conds[0], np.ndarray)):
        conds = [conds]
    conds = [np.asarray(c, dtype=bool) for c in conds]

    return np.piecewise(M_ratio, conds, func_epsilon)

{'fraction_non_alphanumeric': 0.1143955069464972, 'fraction_numerical': 0.03226215109159242, 'mean_word_length': 1.9988603267063443, 'pattern_counts': {'":': 8, '<': 31, '<?xml version=': 0, '>': 35, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4002288', 'n_tokens_mistral': 8700, 'n_tokens_neox': 7941, 'n_words': 2173}