<commit_before>package org.broadinstitute.sting.utils;

import cern.jet.math.Arithmetic;
import cern.jet.random.Normal;
import org.broadinstitute.sting.utils.collections.Pair;
import org.broadinstitute.sting.utils.exceptions.StingException;

import java.util.Comparator;
import java.util.TreeSet;

/**
 * Created by IntelliJ IDEA.
 * User: chartl
 */
public class MannWhitneyU {

    private static Normal STANDARD_NORMAL = new Normal(0.0,1.0,null);

    private TreeSet<Pair<Number,USet>> observations;
    private int sizeSet1;
    private int sizeSet2;

    public MannWhitneyU() {
        observations = new TreeSet<Pair<Number,USet>>(new DitheringComparator());
        sizeSet1 = 0;
        sizeSet2 = 0;
    }

    /**
     * Add an observation into the observation tree
     * @param n: the observation (a number)
     * @param set: whether the observation comes from set 1 or set 2
     */
    public void add(Number n, USet set) {
        observations.add(new Pair<Number,USet>(n,set));
        if ( set == USet.SET1 ) {
            ++sizeSet1;
        } else {
            ++sizeSet2;
        }
    }

    /**
     * temporary method that will be generalized. Runs the standard two-sided test,
     * returns the u and p values.
     * @Returns a pair holding the u and p-value.
     */
    public Pair<Long,Double> runTwoSidedTest() {
        Pair<Long,USet> uPair = calculateTwoSidedU(observations);
        long u = uPair.first;
        int n = uPair.second == USet.SET1 ? sizeSet1 : sizeSet2;
        int m = uPair.second == USet.SET1 ? sizeSet2 : sizeSet1;
        double pval = calculateP(n,m,u,true);
        return new Pair<Long,Double>(u,pval);
    }

    /**
     * Given a u statistic, calculate the p-value associated with it, dispatching to approximations where appropriate
     * @param n - The number of entries in the DOMINATED set
     * @param m - The number of entries in the DOMINANT set
     * @param u - the Mann-Whitney U value
     * @param twoSided - is the test twosided
     * @return the (possibly approximate) p-value associated with the MWU test
     * todo -- there must be an approximation for small m and large n
     */
    public static double calculateP(int n, int m, long u, boolean twoSided) {
        double pval;
        if ( n > 8 && m > 8 ) {
            // large m and n - normal approx
            pval = calculatePNormalApproximation(n,m,u);
        } else if ( n > 4 && m > 7 ) {
            // large m, small n - sum uniform approx
            pval = calculatePUniformApproximation(n,m,u);
        } else if ( n > 5 || m > 5 ) {
            pval = calculatePFromTable(n,m,u);
        } else {
            // small m [possibly small n] - full approx
            pval = calculatePRecursively(n,m,u);
        }

        return twoSided ? 2*pval : pval;
    }

    public static double calculatePFromTable(int n, int m, long u) {
        // todo -- actually use a table for:
        // todo      - n small, m large<commit_msg>Due to the approximation not being well-founded in this case, (and the non-existence of a pre-computed table at this time), pushing up the cutoff<commit_after>package org.broadinstitute.sting.utils;

import cern.jet.math.Arithmetic;
import cern.jet.random.Normal;
import org.broadinstitute.sting.utils.collections.Pair;
import org.broadinstitute.sting.utils.exceptions.StingException;

import java.util.Comparator;
import java.util.TreeSet;

/**
 * Created by IntelliJ IDEA.
 * User: chartl
 */
public class MannWhitneyU {

    private static Normal STANDARD_NORMAL = new Normal(0.0,1.0,null);

    private TreeSet<Pair<Number,USet>> observations;
    private int sizeSet1;
    private int sizeSet2;

    public MannWhitneyU() {
        observations = new TreeSet<Pair<Number,USet>>(new DitheringComparator());
        sizeSet1 = 0;
        sizeSet2 = 0;
    }

    /**
     * Add an observation into the observation tree
     * @param n: the observation (a number)
     * @param set: whether the observation comes from set 1 or set 2
     */
    public void add(Number n, USet set) {
        observations.add(new Pair<Number,USet>(n,set));
        if ( set == USet.SET1 ) {
            ++sizeSet1;
        } else {
            ++sizeSet2;
        }
    }

    /**
     * temporary method that will be generalized. Runs the standard two-sided test,
     * returns the u and p values.
     * @Returns a pair holding the u and p-value.
     */
    public Pair<Long,Double> runTwoSidedTest() {
        Pair<Long,USet> uPair = calculateTwoSidedU(observations);
        long u = uPair.first;
        int n = uPair.second == USet.SET1 ? sizeSet1 : sizeSet2;
        int m = uPair.second == USet.SET1 ? sizeSet2 : sizeSet1;
        double pval = calculateP(n,m,u,true);
        return new Pair<Long,Double>(u,pval);
    }

    /**
     * Given a u statistic, calculate the p-value associated with it, dispatching to approximations where appropriate
     * @param n - The number of entries in the DOMINATED set
     * @param m - The number of entries in the DOMINANT set
     * @param u - the Mann-Whitney U value
     * @param twoSided - is the test twosided
     * @return the (possibly approximate) p-value associated with the MWU test
     * todo -- there must be an approximation for small m and large n
     */
    public static double calculateP(int n, int m, long u, boolean twoSided) {
        double pval;
        if ( n > 8 && m > 8 ) {
            // large m and n - normal approx
            pval = calculatePNormalApproximation(n,m,u);
        } else if ( n > 4 && m > 7 ) {
            // large m, small n - sum uniform approx
            pval = calculatePUniformApproximation(n,m,u);
        } else if ( n > 8 || m > 8 ) {
            pval = calculatePFromTable(n,m,u);
        } else {
            // small m [possibly small n] - full approx
            pval = calculatePRecursively(n,m,u);
        }

        return twoSided ? 2*pval : pval;
    }

    public static double calculatePFromTable(int n, int m, long u) {
        // todo -- actually use a table for:
        // todo      - n small, m large
{'fraction_non_alphanumeric': 0.10029546946815496, 'fraction_numerical': 0.009192383453709783, 'mean_word_length': 2.309614340032591, 'pattern_counts': {'":': 0, '<': 19, '<?xml version=': 0, '>': 31, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6740689', 'n_tokens_mistral': 1910, 'n_tokens_neox': 1845, 'n_words': 604}