<commit_before>package il.ac.technion.ie.experiments.service;

import il.ac.technion.ie.exception.NotImplementedYetException;
import il.ac.technion.ie.experiments.exception.SizeNotEqualException;
import il.ac.technion.ie.experiments.model.BlockWithData;
import il.ac.technion.ie.model.Record;
import il.ac.technion.ie.experiments.model.RecordSplit;
import org.apache.commons.math3.distribution.UniformRealDistribution;
import org.apache.log4j.Logger;

import java.util.ArrayList;
import java.util.List;

/**
 * Created by I062070 on 09/09/2015.
 */
public class FuzzyService {

    private UniformRealDistribution splitBlockProbThresh;
    static final Logger logger = Logger.getLogger(FuzzyService.class);


    public FuzzyService() {
        splitBlockProbThresh = new UniformRealDistribution();
    }


    /**
     * The method split blocks in half if the block inner threshold is lower than the given threshold.
     *
     * @param originalBlocks
     * @param threshold
     */
    public List<BlockWithData> splitBlocks(List<BlockWithData> originalBlocks, double threshold) {
        List<BlockWithData> newBlocks = new ArrayList<>(originalBlocks.size());
        for (BlockWithData origBlock : originalBlocks) {
            double splitProbability = splitBlockProbThresh.sample();
            if (splitProbability < threshold) {
                List<Record> blockOneRecords = new ArrayList<>(originalBlocks.size() / 2);
                List<Record> blockTwoRecords = new ArrayList<>(originalBlocks.size() / 2);
                collectRecordsForSplitedBlocks(origBlock, blockOneRecords, blockTwoRecords);
                newBlocks.add(new BlockWithData(blockOneRecords));
                newBlocks.add(new BlockWithData(blockTwoRecords));
            } else {
                newBlocks.add(origBlock);
            }<commit_msg>Added documentation<commit_after>package il.ac.technion.ie.experiments.service;

import il.ac.technion.ie.exception.NotImplementedYetException;
import il.ac.technion.ie.experiments.exception.SizeNotEqualException;
import il.ac.technion.ie.experiments.model.BlockWithData;
import il.ac.technion.ie.model.Record;
import il.ac.technion.ie.experiments.model.RecordSplit;
import org.apache.commons.math3.distribution.UniformRealDistribution;
import org.apache.log4j.Logger;

import java.util.ArrayList;
import java.util.List;

/**
 * Created by I062070 on 09/09/2015.
 */
public class FuzzyService {

    private UniformRealDistribution splitBlockProbThresh;
    static final Logger logger = Logger.getLogger(FuzzyService.class);


    public FuzzyService() {
        splitBlockProbThresh = new UniformRealDistribution();
    }


    /**
     * The method split blocks in half if the block inner threshold is lower than the given threshold.
     * For each block it randomly sample a split probability from the Uniform distribution~[0,1].
     * If the sampled probability is smaller than the threshold {@code sampledProbability < threshold }then the block is split into two blocks.
     * The records of the block are randomly placed into the new blocks.
     * The True representative of the original block is placed in the two new blocks.
     *
     * @param originalBlocks
     * @param threshold
     */
    public List<BlockWithData> splitBlocks(List<BlockWithData> originalBlocks, double threshold) {
        List<BlockWithData> newBlocks = new ArrayList<>(originalBlocks.size());
        for (BlockWithData origBlock : originalBlocks) {
            double splitProbability = splitBlockProbThresh.sample();
            if (splitProbability < threshold) {
                List<Record> blockOneRecords = new ArrayList<>(originalBlocks.size() / 2);
                List<Record> blockTwoRecords = new ArrayList<>(originalBlocks.size() / 2);
                collectRecordsForSplitedBlocks(origBlock, blockOneRecords, blockTwoRecords);
                newBlocks.add(new BlockWithData(blockOneRecords));
                newBlocks.add(new BlockWithData(blockTwoRecords));
            } else {
                newBlocks.add(origBlock);
            }
{'fraction_non_alphanumeric': 0.08972787447903897, 'fraction_numerical': 0.009316008825692572, 'mean_word_length': 3.396551724137931, 'pattern_counts': {'":': 0, '<': 22, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6725121', 'n_tokens_mistral': 1135, 'n_tokens_neox': 1059, 'n_words': 262}