<commit_before>package com.intellij.concurrency;

import com.intellij.openapi.diagnostic.Logger;
import com.intellij.openapi.progress.ProcessCanceledException;
import com.intellij.openapi.progress.ProgressIndicator;
import com.intellij.openapi.progress.ProgressManager;
import com.intellij.openapi.progress.util.ProgressWrapper;
import com.intellij.util.Processor;
import org.jetbrains.annotations.NonNls;
import org.jetbrains.annotations.NotNull;

import java.util.Collection;

/**
 * @author cdr
 */

public class JobUtil {
  private static final Logger LOG = Logger.getInstance("#com.intellij.concurrency.JobUtil");

  /**
   * @param things to process concurrently
   * @param thingProcessor to be invoked concurrently on each element from the collection
   * @param jobName the name of the job that invokes all the tasks
   * @return false if tasks have been canceled
   * @throws ProcessCanceledException if at least one task has thrown ProcessCanceledException
   */
  public static <T> boolean invokeConcurrentlyForAll(@NotNull Collection<T> things, @NotNull final Processor<T> thingProcessor, @NotNull @NonNls String jobName) throws ProcessCanceledException {
    if (things.isEmpty()) {
      return true;
    }
    if (things.size() == 1) {
      T t = things.iterator().next();
      return thingProcessor.process(t);
    }

    final Job<?> job = JobScheduler.getInstance().createJob(jobName, Job.DEFAULT_PRIORITY);

    for (final T thing : things) {
      job.addTask(new Runnable(){
        public void run() {
          if (!thingProcessor.process(thing)) {
            job.cancel();
          }
        }
      });
    }
    try {
      job.scheduleAndWaitForResults();
    }
    catch (RuntimeException e) {
      throw e;
    }
    catch (Throwable throwable) {
      LOG.error(throwable);
    }
    return !job.isCanceled();
  }

  // execute in multiple threads, with checkCanceled in each delegated to our current progress
  public static <T> boolean invokeConcurrentlyUnderMyProgress(@NotNull Collection<T> things, @NotNull final Processor<T> thingProcessor, @NotNull @NonNls String jobName) throws ProcessCanceledException {
    final ProgressIndicator indicator = ProgressManager.getInstance().getProgressIndicator();
    return invokeConcurrentlyForAll(things, new Processor<T>() {
      public boolean process(final T t) {
        final boolean[] result = new boolean[1];
        ProgressManager.getInstance().runProcess(new Runnable() {
          public void run() {
            result[0] = thingProcessor.process(t);
          }
        }, ProgressWrapper.wrap(indicator));
        return result[0];
      }
    }, jobName);
  }

}
<commit_msg>diagnostics for race<commit_after>package com.intellij.concurrency;

import com.intellij.openapi.diagnostic.Logger;
import com.intellij.openapi.progress.ProcessCanceledException;
import com.intellij.openapi.progress.ProgressIndicator;
import com.intellij.openapi.progress.ProgressManager;
import com.intellij.openapi.progress.util.ProgressWrapper;
import com.intellij.util.Processor;
import org.jetbrains.annotations.NonNls;
import org.jetbrains.annotations.NotNull;

import java.util.Collection;

/**
 * @author cdr
 */

public class JobUtil {
  private static final Logger LOG = Logger.getInstance("#com.intellij.concurrency.JobUtil");

  /**
   * @param things to process concurrently
   * @param thingProcessor to be invoked concurrently on each element from the collection
   * @param jobName the name of the job that invokes all the tasks
   * @return false if tasks have been canceled
   * @throws ProcessCanceledException if at least one task has thrown ProcessCanceledException
   */
  public static <T> boolean invokeConcurrentlyForAll(@NotNull Collection<T> things, @NotNull final Processor<T> thingProcessor, @NotNull @NonNls String jobName) throws ProcessCanceledException {
    if (things.isEmpty()) {
      return true;
    }
    if (things.size() == 1) {
      T t = things.iterator().next();
      return thingProcessor.process(t);
    }

    final Job<String> job = JobScheduler.getInstance().createJob(jobName, Job.DEFAULT_PRIORITY);

    for (final T thing : things) {
      //noinspection HardCodedStringLiteral
      job.addTask(new Runnable(){
        public void run() {
          if (!thingProcessor.process(thing)) {
            job.cancel();
          }
        }
      }, "done");
    }
    try {
      job.scheduleAndWaitForResults();
    }
    catch (RuntimeException e) {
      throw e;
    }
    catch (Throwable throwable) {
      LOG.error(throwable);
    }
    return !job.isCanceled();
  }

  // execute in multiple threads, with checkCanceled in each delegated to our current progress
  public static <T> boolean invokeConcurrentlyUnderMyProgress(@NotNull Collection<T> things, @NotNull final Processor<T> thingProcessor, @NotNull @NonNls String jobName) throws ProcessCanceledException {
    final ProgressIndicator indicator = ProgressManager.getInstance().getProgressIndicator();
    return invokeConcurrentlyForAll(things, new Processor<T>() {
      public boolean process(final T t) {
        final boolean[] result = new boolean[1];
        ProgressManager.getInstance().runProcess(new Runnable() {
          public void run() {
            result[0] = thingProcessor.process(t);
          }
        }, ProgressWrapper.wrap(indicator));
        return result[0];
      }
    }, jobName);
  }

}

{'fraction_non_alphanumeric': 0.10051736881005174, 'fraction_numerical': 0.0014781966001478197, 'mean_word_length': 3.5108333333333333, 'pattern_counts': {'":': 0, '<': 19, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4738550', 'n_tokens_mistral': 1512, 'n_tokens_neox': 1492, 'n_words': 397}