<commit_before>/*
 * Copyright 2000-2009 JetBrains s.r.o.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package com.intellij.openapi.project;

import com.intellij.concurrency.Job;
import com.intellij.concurrency.JobScheduler;
import com.intellij.ide.caches.CacheUpdater;
import com.intellij.ide.caches.FileContent;
import com.intellij.openapi.application.Application;
import com.intellij.openapi.application.ApplicationAdapter;
import com.intellij.openapi.application.ApplicationManager;
import com.intellij.openapi.diagnostic.Logger;
import com.intellij.openapi.progress.ProcessCanceledException;
import com.intellij.openapi.progress.ProgressIndicator;
import com.intellij.openapi.progress.ProgressManager;
import com.intellij.openapi.progress.util.ProgressIndicatorBase;
import com.intellij.openapi.util.Ref;
import com.intellij.openapi.util.registry.Registry;
import com.intellij.openapi.vfs.VirtualFile;
import com.intellij.util.Consumer;
import gnu.trove.THashSet;

import java.util.Collection;
import java.util.Set;

class CacheUpdateRunner {
  private static final Logger LOG = Logger.getInstance("#com.intellij.openapi.project.CacheUpdateRunner");
  private final Project myProject;
  private final Collection<CacheUpdater> myUpdaters;
  private CacheUpdateSession mySession;

  CacheUpdateRunner(Project project, Collection<CacheUpdater> updaters) {
    myProject = project;
    myUpdaters = updaters;
  }

  public int queryNeededFiles(ProgressIndicator indicator) {
    // can be queried twice in DumbService  
    if (mySession == null) {
      mySession = new CacheUpdateSession(myUpdaters, indicator);
    }
    return mySession.getFilesToUpdate().size();
  }

  public int getNumberOfPendingUpdateJobs(ProgressIndicator indicator) {
    if (mySession == null) {
      mySession = new CacheUpdateSession(myUpdaters, indicator);
    }
    return mySession.getNumberOfPendingUpdateJobs();
  }

  public void processFiles(final ProgressIndicator indicator, boolean processInReadAction) {
    try {
      indicator.checkCanceled();
      final FileContentQueue queue = new FileContentQueue();
      Collection<VirtualFile> files = mySession.getFilesToUpdate();
      final double total = files.size();
      queue.queue(files, indicator);

      Consumer<VirtualFile> progressUpdater = new Consumer<VirtualFile>() {
        // need set here to handle queue.pushbacks after checkCancelled() in order
        // not to count the same file several times
        final Set<VirtualFile> processed = new THashSet<VirtualFile>();

        public void consume(VirtualFile virtualFile) {
          indicator.checkCanceled();
          synchronized (processed) {
            processed.add(virtualFile);
            indicator.setFraction(processed.size() / total);
          }
          if (virtualFile.isValid()) {
            indicator.setText2(virtualFile.getPresentableUrl());
          }
          else {
            indicator.setText2("");
          }
        }
      };

      while (!myProject.isDisposed()) {
        indicator.checkCanceled();
        // todo wait for the user...
        if (processSomeFilesWhileUserIsInactive(queue, progressUpdater, processInReadAction)) {
          break;
        }
      }

      if (myProject.isDisposed()) {
        indicator.cancel();
        indicator.checkCanceled();
      }
    }
    catch (ProcessCanceledException e) {
      mySession.canceled();
      throw e;
    }
  }

  public void updatingDone() {
    try {
      mySession.updatingDone();
    }
    catch (ProcessCanceledException e) {
      mySession.canceled();
      throw e;
    }
  }

  private boolean processSomeFilesWhileUserIsInactive(final FileContentQueue queue,
                                                      final Consumer<VirtualFile> progressUpdater,
                                                      final boolean processInReadAction) {
    final ProgressIndicatorBase innerIndicator = new ProgressIndicatorBase();
    final ApplicationAdapter canceller = new ApplicationAdapter() {
      @Override
      public void beforeWriteActionStart(Object action) {
        innerIndicator.cancel();
      }
    };
    final Application application = ApplicationManager.getApplication();
    application.addApplicationListener(canceller);

    final Ref<Boolean> isFinished = new Ref<Boolean>(Boolean.FALSE);
    try {
      int threadsCount = Registry.intValue("caches.indexerThreadsCount");
      if (threadsCount == -1) {
        threadsCount = Math.min(Runtime.getRuntime().availableProcessors(), ApplicationManager.getApplication().isUnitTestMode() ? 1 : 2);
      }
      if (threadsCount == 1) {
        Runnable process = new MyRunnable(innerIndicator, queue, isFinished, progressUpdater, processInReadAction, application);
        ProgressManager.getInstance().runProcess(process, innerIndicator);
      }
      else {
        final Job<Object> job = JobScheduler.getInstance().createJob("Indexing", Thread.NORM_PRIORITY);
        final Ref[] finishedRefs = new Ref[threadsCount];
        for (int i = 0; i < threadsCount; i++) {
          final Ref<Boolean> ref = new Ref<Boolean>(Boolean.FALSE);
          finishedRefs[i] = ref;
          Runnable process = new MyRunnable(innerIndicator, queue, ref, progressUpdater, processInReadAction, application);
          job.addTask(process);
        }
        try {
          job.scheduleAndWaitForResults();
          
          boolean allFinished = true;
          for (Ref ref : finishedRefs) {
            if (!(Boolean)ref.get()) {
              allFinished = false;
              break;
            }
          }<commit_msg>disable multithreaded indexing<commit_after>/*
 * Copyright 2000-2009 JetBrains s.r.o.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package com.intellij.openapi.project;

import com.intellij.concurrency.Job;
import com.intellij.concurrency.JobScheduler;
import com.intellij.ide.caches.CacheUpdater;
import com.intellij.ide.caches.FileContent;
import com.intellij.openapi.application.Application;
import com.intellij.openapi.application.ApplicationAdapter;
import com.intellij.openapi.application.ApplicationManager;
import com.intellij.openapi.diagnostic.Logger;
import com.intellij.openapi.progress.ProcessCanceledException;
import com.intellij.openapi.progress.ProgressIndicator;
import com.intellij.openapi.progress.ProgressManager;
import com.intellij.openapi.progress.util.ProgressIndicatorBase;
import com.intellij.openapi.util.Ref;
import com.intellij.openapi.util.registry.Registry;
import com.intellij.openapi.vfs.VirtualFile;
import com.intellij.util.Consumer;
import gnu.trove.THashSet;

import java.util.Collection;
import java.util.Set;

class CacheUpdateRunner {
  private static final Logger LOG = Logger.getInstance("#com.intellij.openapi.project.CacheUpdateRunner");
  private final Project myProject;
  private final Collection<CacheUpdater> myUpdaters;
  private CacheUpdateSession mySession;

  CacheUpdateRunner(Project project, Collection<CacheUpdater> updaters) {
    myProject = project;
    myUpdaters = updaters;
  }

  public int queryNeededFiles(ProgressIndicator indicator) {
    // can be queried twice in DumbService  
    if (mySession == null) {
      mySession = new CacheUpdateSession(myUpdaters, indicator);
    }
    return mySession.getFilesToUpdate().size();
  }

  public int getNumberOfPendingUpdateJobs(ProgressIndicator indicator) {
    if (mySession == null) {
      mySession = new CacheUpdateSession(myUpdaters, indicator);
    }
    return mySession.getNumberOfPendingUpdateJobs();
  }

  public void processFiles(final ProgressIndicator indicator, boolean processInReadAction) {
    try {
      indicator.checkCanceled();
      final FileContentQueue queue = new FileContentQueue();
      Collection<VirtualFile> files = mySession.getFilesToUpdate();
      final double total = files.size();
      queue.queue(files, indicator);

      Consumer<VirtualFile> progressUpdater = new Consumer<VirtualFile>() {
        // need set here to handle queue.pushbacks after checkCancelled() in order
        // not to count the same file several times
        final Set<VirtualFile> processed = new THashSet<VirtualFile>();

        public void consume(VirtualFile virtualFile) {
          indicator.checkCanceled();
          synchronized (processed) {
            processed.add(virtualFile);
            indicator.setFraction(processed.size() / total);
          }
          if (virtualFile.isValid()) {
            indicator.setText2(virtualFile.getPresentableUrl());
          }
          else {
            indicator.setText2("");
          }
        }
      };

      while (!myProject.isDisposed()) {
        indicator.checkCanceled();
        // todo wait for the user...
        if (processSomeFilesWhileUserIsInactive(queue, progressUpdater, processInReadAction)) {
          break;
        }
      }

      if (myProject.isDisposed()) {
        indicator.cancel();
        indicator.checkCanceled();
      }
    }
    catch (ProcessCanceledException e) {
      mySession.canceled();
      throw e;
    }
  }

  public void updatingDone() {
    try {
      mySession.updatingDone();
    }
    catch (ProcessCanceledException e) {
      mySession.canceled();
      throw e;
    }
  }

  private boolean processSomeFilesWhileUserIsInactive(final FileContentQueue queue,
                                                      final Consumer<VirtualFile> progressUpdater,
                                                      final boolean processInReadAction) {
    final ProgressIndicatorBase innerIndicator = new ProgressIndicatorBase();
    final ApplicationAdapter canceller = new ApplicationAdapter() {
      @Override
      public void beforeWriteActionStart(Object action) {
        innerIndicator.cancel();
      }
    };
    final Application application = ApplicationManager.getApplication();
    application.addApplicationListener(canceller);

    final Ref<Boolean> isFinished = new Ref<Boolean>(Boolean.FALSE);
    try {
      int threadsCount = Registry.intValue("caches.indexerThreadsCount");
      if (threadsCount == -1) {
        threadsCount = Math.min(Runtime.getRuntime().availableProcessors(), ApplicationManager.getApplication().isUnitTestMode() ? 1 : 2);
      }
      if (true) {
        Runnable process = new MyRunnable(innerIndicator, queue, isFinished, progressUpdater, processInReadAction, application);
        ProgressManager.getInstance().runProcess(process, innerIndicator);
      }
      else {
        final Job<Object> job = JobScheduler.getInstance().createJob("Indexing", Thread.NORM_PRIORITY);
        final Ref[] finishedRefs = new Ref[threadsCount];
        for (int i = 0; i < threadsCount; i++) {
          final Ref<Boolean> ref = new Ref<Boolean>(Boolean.FALSE);
          finishedRefs[i] = ref;
          Runnable process = new MyRunnable(innerIndicator, queue, ref, progressUpdater, processInReadAction, application);
          job.addTask(process);
        }
        try {
          job.scheduleAndWaitForResults();
          
          boolean allFinished = true;
          for (Ref ref : finishedRefs) {
            if (!(Boolean)ref.get()) {
              allFinished = false;
              break;
            }
          }
{'fraction_non_alphanumeric': 0.09267417196932616, 'fraction_numerical': 0.003018436939141785, 'mean_word_length': 3.0579278384640847, 'pattern_counts': {'":': 0, '<': 31, '<?xml version=': 0, '>': 29, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7490510', 'n_tokens_mistral': 3271, 'n_tokens_neox': 3281, 'n_words': 787}