<commit_before>package peergos.user.fs.erasure;

import org.junit.*;

import java.math.*;
import java.util.*;
import java.util.function.*;
import java.util.stream.*;

public class ErrorTests {
    public static class ErasureParameters {
        @org.junit.Test
        public void findAcceptableErasureParameters() {
            int totalUserSizeInMB = 10000;
            int nChunks = totalUserSizeInMB / 5;
            Predicate<BigDecimal> acceptableFailure = pr -> pr.doubleValue() < 1.0/nChunks; // Expect not to lose a single chunk
            Stream.of(60).forEach(n -> IntStream.range(1, 6).map(i -> i*n/6)
                    .forEach(k -> Stream.of(0.5, 0.7, 0.8, 0.9).forEach(p -> {
                        BigDecimal prFail = probabilityFailure(n, k, p);
                        if (acceptableFailure.test(prFail))
                            System.out.printf("%d, %d, %.2f -> %.15f\n", n, k, p, prFail);
                    }
                    )
                    )
            );

            double p = 0.7;
            int f = 40, e = 30;
            int n = f + 2*e, k = f + e;
            while (!acceptableFailure.test(probabilityFailure(n, k, p)))
                p = (1 + p)/2;
            System.out.printf("%d, %d, %.2f -> %.15f\n", n, k, p, probabilityFailure(n, k, p));
        }
    }

    /*
    *  The probability of data loss using a k of n erasure code with each fragment having pr(correct) = p
    * */
    public static BigDecimal probabilityFailure(int n, int k, double p) {
        List<BigDecimal> collect = IntStream.range(0, k)
                .mapToObj(i -> new BigDecimal(p).pow(i).multiply(new BigDecimal(1 - p).pow(n - i)).multiply(choose(n, i))).collect(Collectors.toList());
        return collect.stream()
        .reduce(new BigDecimal(0), (a, b) -> a.add(b));
    }

    static BigInteger[][] choose = new BigInteger[200][200];
    static {
        for (int i=0; i < choose.length; i++) {
            choose[i][0] = BigInteger.valueOf(1);
            choose[i][i] = BigInteger.valueOf(1);
        }
        for (int i=1; i < choose.length; i++)
            for (int j=1; j < i; j++)
                choose[i][j] = choose[i-1][j-1].add(choose[i-1][j]);
    }<commit_msg>Find required online % for successful erasure success rate<commit_after>package peergos.user.fs.erasure;

import org.junit.*;

import java.math.*;
import java.util.*;
import java.util.function.*;
import java.util.stream.*;

public class ErrorTests {
    public static class ErasureParameters {
        @org.junit.Test
        public void findAcceptableErasureParameters() {
            int totalUserSizeInMB = 10000;
            int nChunks = totalUserSizeInMB / 5;
            Predicate<BigDecimal> acceptableFailure = pr -> pr.doubleValue() < 1.0/nChunks; // Expect not to lose a single chunk
            Stream.of(60).forEach(n -> IntStream.range(1, 6).map(i -> i*n/6)
                    .forEach(k -> Stream.of(0.5, 0.7, 0.8, 0.9).forEach(p -> {
                        BigDecimal prFail = probabilityFailure(n, k, p);
                        if (acceptableFailure.test(prFail))
                            System.out.printf("%d, %d, %.2f -> %.15f\n", n, k, p, prFail);
                    }
                    )
                    )
            );

            double p = 0.5;
            int f = 40, e = 10;
            int n = f + 2*e, k = f + e;

            double min_p = 0.0, max_p = 1.0;
            while (true) {
                if (acceptableFailure.test(probabilityFailure(n, k, p))) {
                    max_p = p;
                    p = (p + min_p) / 2;
                } else {
                    min_p = p;
                    p = (1 + p) / 2;
                }
                if (Math.abs(max_p - min_p) < 0.01)
                    break;
            }
            System.out.printf("%d, %d, %.2f -> %.15f\n", n, k, p, probabilityFailure(n, k, p));
        }
    }

    /*
    *  The probability of data loss using a k of n erasure code with each fragment having pr(correct) = p
    * */
    public static BigDecimal probabilityFailure(int n, int k, double p) {
        List<BigDecimal> collect = IntStream.range(0, k)
                .mapToObj(i -> new BigDecimal(p).pow(i).multiply(new BigDecimal(1 - p).pow(n - i)).multiply(choose(n, i))).collect(Collectors.toList());
        return collect.stream()
        .reduce(new BigDecimal(0), (a, b) -> a.add(b));
    }

    static BigInteger[][] choose = new BigInteger[200][200];
    static {
        for (int i=0; i < choose.length; i++) {
            choose[i][0] = BigInteger.valueOf(1);
            choose[i][i] = BigInteger.valueOf(1);
        }
        for (int i=1; i < choose.length; i++)
            for (int j=1; j < i; j++)
                choose[i][j] = choose[i-1][j-1].add(choose[i-1][j]);
    }
{'fraction_non_alphanumeric': 0.14838709677419354, 'fraction_numerical': 0.024141519250780438, 'mean_word_length': 1.667036625971143, 'pattern_counts': {'":': 0, '<': 16, '<?xml version=': 0, '>': 25, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5385388', 'n_tokens_mistral': 1591, 'n_tokens_neox': 1492, 'n_words': 375}