<commit_before>package ca.qc.bergeron.marcantoine.crammeur.librairy.utils;

import java.util.Collection;
import java.util.Iterator;
import java.util.NoSuchElementException;
import java.util.concurrent.Callable;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;

/**
 * Created by Marc-Antoine on 2017-09-24.
 */

public final class Parallel {

    private static final int NUM_CORES = Runtime.getRuntime().availableProcessors();
    private static final int MAX_THREAD = NUM_CORES*2;

    public static <T> void For(final Collection<T> elements, final Operation<T> operation) {
        ExecutorService executor = Executors.newFixedThreadPool(MAX_THREAD);
        final Iterator<T> iterator = elements.iterator();
        final Runnable runnable = new Runnable() {
            final Callable<Void> callable = new Callable<Void>() {
                @Override
                public Void call() throws Exception {
                    T result;
                    synchronized (iterator) {
                        result = iterator.next();
                    }
                    operation.perform(result);
                    return null;
                }
            };
            @Override
            public void run() {
                while (iterator.hasNext()) {
                    try {
                        synchronized (operation) {
                            if (operation.follow()) {
                                synchronized (callable) {
                                    callable.call();
                                }
                                if (!operation.follow()) {
                                    break;
                                }
                            } else {
                                break;
                            }
                        }
                    } catch (NoSuchElementException e) {
                        break;
                    } catch (Exception e) {
                        e.printStackTrace();
                        throw new RuntimeException(e);
                    }
                }
            }
        };
        for (int threadIndex=0; threadIndex<MAX_THREAD; threadIndex++) {
            executor.execute(runnable);
        }
        executor.shutdown();
        while (!executor.isTerminated()) {
            try {
                Thread.sleep(0,1);
            } catch (InterruptedException e) {
                e.printStackTrace();
                throw new RuntimeException(e);
            }
        }
    }

    public static <T> void For(final T[] elements, final Operation<T> operation) {
        ExecutorService executor = Executors.newFixedThreadPool(MAX_THREAD);
        final int[] index = new int[1];
        final Runnable runnable = new Runnable() {
            final Callable<Void> callable = new Callable<Void>() {
                @Override
                public Void call() throws Exception {
                    T result = elements[index[0]];
                    operation.perform(result);
                    return null;
                }
            };
            @Override
            public void run() {
                while (index[0] < elements.length) {
                    try {
                        synchronized (operation) {
                            if (operation.follow()) {
                                synchronized (callable) {
                                    callable.call();
                                }
                                if (!operation.follow()) {
                                    break;
                                }
                            } else {
                                break;
                            }
                        }
                    } catch (NoSuchElementException e) {
                        break;
                    } catch (Exception e) {
                        e.printStackTrace();
                        throw new RuntimeException(e);
                    }
                }
            }
        };
        for (int threadIndex=0; threadIndex<MAX_THREAD; threadIndex++) {
            executor.execute(runnable);
        }
        executor.shutdown();
        while (!executor.isTerminated()) {
            try {
                Thread.sleep(0,1);
            } catch (InterruptedException e) {
                e.printStackTrace();
                throw new RuntimeException(e);
            }
        }
    }
<commit_msg>first commit<commit_after>package ca.qc.bergeron.marcantoine.crammeur.librairy.utils;

import java.util.Collection;
import java.util.Iterator;
import java.util.NoSuchElementException;
import java.util.concurrent.Callable;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;

/**
 * Created by Marc-Antoine on 2017-09-24.
 */

public final class Parallel {

    private static final int NUM_CORES = Runtime.getRuntime().availableProcessors();
    private static final int MAX_THREAD = NUM_CORES*2;

    public static <T> void For(final Collection<T> elements, final Operation<T> operation) {
        ExecutorService executor = Executors.newFixedThreadPool(MAX_THREAD);
        final Iterator<T> iterator = elements.iterator();
        final Runnable runnable = new Runnable() {
            final Callable<Void> callable = new Callable<Void>() {
                @Override
                public Void call() throws Exception {
                    T result;
                    synchronized (iterator) {
                        result = iterator.next();
                    }
                    operation.perform(result);
                    return null;
                }
            };
            @Override
            public void run() {
                while (iterator.hasNext()) {
                    try {
                        synchronized (operation) {
                            if (operation.follow()) {
                                synchronized (callable) {
                                    callable.call();
                                }
                                if (!operation.follow()) {
                                    break;
                                }
                            } else {
                                break;
                            }
                        }
                    } catch (NoSuchElementException e) {
                        break;
                    } catch (Exception e) {
                        e.printStackTrace();
                        throw new RuntimeException(e);
                    }
                }
            }
        };
        for (int threadIndex=0; threadIndex<MAX_THREAD; threadIndex++) {
            executor.execute(runnable);
        }
        executor.shutdown();
        while (!executor.isTerminated()) {
            try {
                Thread.sleep(0,1);
            } catch (InterruptedException e) {
                e.printStackTrace();
                throw new RuntimeException(e);
            }
        }
    }

    public static <T> void For(final T[] elements, final Operation<T> operation) {
        ExecutorService executor = Executors.newFixedThreadPool(MAX_THREAD);
        final int[] index = new int[1];
        final Runnable runnable = new Runnable() {
            final Callable<Void> callable = new Callable<Void>() {
                @Override
                public Void call() throws Exception {
                    T result = elements[index[0]];
                    operation.perform(result);
                    return null;
                }
            };
            @Override
            public void run() {
                while (index[0] < elements.length) {
                    try {
                        synchronized (operation) {
                            if (operation.follow()) {
                                synchronized (callable) {
                                    callable.call();
                                }
                                synchronized (index) {
                                    index[0]++;
                                }
                                if (!operation.follow()) {
                                    break;
                                }
                            } else {
                                break;
                            }
                        }
                    } catch (NoSuchElementException e) {
                        break;
                    } catch (Exception e) {
                        e.printStackTrace();
                        throw new RuntimeException(e);
                    }
                }
            }
        };
        for (int threadIndex=0; threadIndex<MAX_THREAD; threadIndex++) {
            executor.execute(runnable);
        }
        executor.shutdown();
        while (!executor.isTerminated()) {
            try {
                Thread.sleep(0,1);
            } catch (InterruptedException e) {
                e.printStackTrace();
                throw new RuntimeException(e);
            }
        }
    }

{'fraction_non_alphanumeric': 0.08375411635565312, 'fraction_numerical': 0.0040614709110867175, 'mean_word_length': 1.000219538968167, 'pattern_counts': {'":': 0, '<': 29, '<?xml version=': 0, '>': 23, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 13, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4945373', 'n_tokens_mistral': 2081, 'n_tokens_neox': 1951, 'n_words': 432}