<commit_before>package com.intellij.util.indexing;

import com.intellij.ide.startup.CacheUpdater;
import com.intellij.ide.startup.FileContent;
import com.intellij.ide.startup.FileContentQueue;
import com.intellij.openapi.application.ApplicationAdapter;
import com.intellij.openapi.application.ApplicationManager;
import com.intellij.openapi.diagnostic.Logger;
import com.intellij.openapi.progress.EmptyProgressIndicator;
import com.intellij.openapi.progress.ProcessCanceledException;
import com.intellij.openapi.progress.ProgressIndicator;
import com.intellij.openapi.progress.ProgressManager;
import com.intellij.openapi.progress.util.ProgressIndicatorBase;
import com.intellij.openapi.project.DumbServiceImpl;
import com.intellij.openapi.project.Project;
import com.intellij.openapi.project.ProjectUtil;
import com.intellij.openapi.util.Key;
import com.intellij.openapi.util.Ref;
import com.intellij.openapi.vfs.VirtualFile;
import com.intellij.util.Consumer;
import com.intellij.util.containers.HashMap;
import gnu.trove.THashSet;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;

import java.util.*;

/**
 * @author <NAME>
 *         Date: Sep 22, 2009
 */
public class BackgroundCacheUpdaterRunner {
  private static final Logger LOG = Logger.getInstance("#com.intellij.util.indexing.BackgroundCacheUpdaterRunner");
  private static final Key<Boolean> DONT_INDEX_AGAIN_KEY = Key.create("DONT_INDEX_AGAIN_KEY");
  private final Map<Project, Collection<VirtualFile>> myProjectToFileMap;  // todo: --> Map<Project, VirtualFile>

  public BackgroundCacheUpdaterRunner(Project project, Collection<VirtualFile> files) {
    myProjectToFileMap = Collections.singletonMap(project, files);
  }

  public BackgroundCacheUpdaterRunner(Collection<VirtualFile> files) {
    myProjectToFileMap = new HashMap<Project, Collection<VirtualFile>>();
    for (VirtualFile file : files) {
      final Project project = ProjectUtil.guessProjectForFile(file);
      Collection<VirtualFile> list = myProjectToFileMap.get(project);
      if (list == null) {
        myProjectToFileMap.put(project, list = new ArrayList<VirtualFile>());
      }
      list.add(file);
    }
  }

  public void processFiles(final CacheUpdater updater) {
    for (Map.Entry<Project, Collection<VirtualFile>> entry : myProjectToFileMap.entrySet()) {
      final Project project = entry.getKey();
      final Collection<VirtualFile> files = entry.getValue();

      final Consumer<ProgressIndicator> action = new Consumer<ProgressIndicator>() {
        public void consume(final ProgressIndicator indicator) {
          final MyFileContentQueue queue = new MyFileContentQueue();

          try {
            final double count = files.size();
            queue.queue(files, indicator);

            final Consumer<VirtualFile> uiUpdater = new Consumer<VirtualFile>() {
              final Set<VirtualFile> processed = new THashSet<VirtualFile>();

              public void consume(VirtualFile virtualFile) {
                indicator.checkCanceled();
                indicator.setFraction(processed.size() / count);
                processed.add(virtualFile);
                indicator.setText2(virtualFile.getPresentableUrl());
              }
            };

            while (!project.isDisposed()) {
              indicator.checkCanceled();
              if (runWhileUserInactive(project, queue, uiUpdater, updater)) {
                break;
              }
            }
            if (project.isDisposed()) {
              indicator.cancel();
            }
          }
          finally {
            queue.clear();
            updater.updatingDone();
          }
        }
      };
      if (project != null) {
        DumbServiceImpl.getInstance(project).queueIndexUpdate(action);
      }
      else {
        final ProgressIndicator currentIndicator = ProgressManager.getInstance().getProgressIndicator();
        action.consume(currentIndicator != null? currentIndicator : new EmptyProgressIndicator());<commit_msg>fix NPE<commit_after>package com.intellij.util.indexing;

import com.intellij.ide.startup.CacheUpdater;
import com.intellij.ide.startup.FileContent;
import com.intellij.ide.startup.FileContentQueue;
import com.intellij.openapi.application.ApplicationAdapter;
import com.intellij.openapi.application.ApplicationManager;
import com.intellij.openapi.diagnostic.Logger;
import com.intellij.openapi.progress.EmptyProgressIndicator;
import com.intellij.openapi.progress.ProcessCanceledException;
import com.intellij.openapi.progress.ProgressIndicator;
import com.intellij.openapi.progress.ProgressManager;
import com.intellij.openapi.progress.util.ProgressIndicatorBase;
import com.intellij.openapi.project.DumbServiceImpl;
import com.intellij.openapi.project.Project;
import com.intellij.openapi.project.ProjectUtil;
import com.intellij.openapi.util.Key;
import com.intellij.openapi.util.Ref;
import com.intellij.openapi.vfs.VirtualFile;
import com.intellij.util.Consumer;
import com.intellij.util.containers.HashMap;
import gnu.trove.THashSet;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;

import java.util.*;

/**
 * @author <NAME>
 *         Date: Sep 22, 2009
 */
public class BackgroundCacheUpdaterRunner {
  private static final Logger LOG = Logger.getInstance("#com.intellij.util.indexing.BackgroundCacheUpdaterRunner");
  private static final Key<Boolean> DONT_INDEX_AGAIN_KEY = Key.create("DONT_INDEX_AGAIN_KEY");
  private final Map<Project, Collection<VirtualFile>> myProjectToFileMap;  // todo: --> Map<Project, VirtualFile>

  public BackgroundCacheUpdaterRunner(Project project, Collection<VirtualFile> files) {
    myProjectToFileMap = Collections.singletonMap(project, files);
  }

  public BackgroundCacheUpdaterRunner(Collection<VirtualFile> files) {
    myProjectToFileMap = new HashMap<Project, Collection<VirtualFile>>();
    for (VirtualFile file : files) {
      final Project project = ProjectUtil.guessProjectForFile(file);
      Collection<VirtualFile> list = myProjectToFileMap.get(project);
      if (list == null) {
        myProjectToFileMap.put(project, list = new ArrayList<VirtualFile>());
      }
      list.add(file);
    }
  }

  public void processFiles(final CacheUpdater updater) {
    for (Map.Entry<Project, Collection<VirtualFile>> entry : myProjectToFileMap.entrySet()) {
      final Project project = entry.getKey();
      final Collection<VirtualFile> files = entry.getValue();

      final Consumer<ProgressIndicator> action = new Consumer<ProgressIndicator>() {
        public void consume(final ProgressIndicator indicator) {
          final MyFileContentQueue queue = new MyFileContentQueue();

          try {
            final double count = files.size();
            queue.queue(files, indicator);

            final Consumer<VirtualFile> uiUpdater = new Consumer<VirtualFile>() {
              final Set<VirtualFile> processed = new THashSet<VirtualFile>();

              public void consume(VirtualFile virtualFile) {
                indicator.checkCanceled();
                indicator.setFraction(processed.size() / count);
                processed.add(virtualFile);
                indicator.setText2(virtualFile.getPresentableUrl());
              }
            };

            while (project == null || !project.isDisposed()) {
              indicator.checkCanceled();
              if (runWhileUserInactive(project, queue, uiUpdater, updater)) {
                break;
              }
            }
            if (project != null && project.isDisposed()) {
              indicator.cancel();
            }
          }
          finally {
            queue.clear();
            updater.updatingDone();
          }
        }
      };
      if (project != null) {
        DumbServiceImpl.getInstance(project).queueIndexUpdate(action);
      }
      else {
        final ProgressIndicator currentIndicator = ProgressManager.getInstance().getProgressIndicator();
        action.consume(currentIndicator != null? currentIndicator : new EmptyProgressIndicator());
{'fraction_non_alphanumeric': 0.10230369085954917, 'fraction_numerical': 0.0017339608620262571, 'mean_word_length': 3.3885869565217392, 'pattern_counts': {'":': 0, '<': 43, '<?xml version=': 0, '>': 45, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '262856', 'n_tokens_mistral': 2210, 'n_tokens_neox': 2217, 'n_words': 402}