<commit_before>/*
 * Copyright 2000-2016 JetBrains s.r.o.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package com.intellij.psi.impl;

import com.intellij.lang.ASTNode;
import com.intellij.lang.FileASTNode;
import com.intellij.openapi.Disposable;
import com.intellij.openapi.application.*;
import com.intellij.openapi.application.ex.ApplicationEx;
import com.intellij.openapi.components.ServiceManager;
import com.intellij.openapi.diagnostic.Logger;
import com.intellij.openapi.editor.Document;
import com.intellij.openapi.editor.ex.DocumentEx;
import com.intellij.openapi.progress.ProcessCanceledException;
import com.intellij.openapi.progress.ProgressIndicator;
import com.intellij.openapi.progress.ProgressManager;
import com.intellij.openapi.progress.util.StandardProgressIndicatorBase;
import com.intellij.openapi.project.Project;
import com.intellij.openapi.util.*;
import com.intellij.openapi.util.text.StringUtil;
import com.intellij.openapi.vfs.VirtualFile;
import com.intellij.pom.PomManager;
import com.intellij.pom.PomModel;
import com.intellij.pom.event.PomModelEvent;
import com.intellij.pom.impl.PomTransactionBase;
import com.intellij.pom.tree.TreeAspect;
import com.intellij.pom.tree.TreeAspectEvent;
import com.intellij.psi.*;
import com.intellij.psi.codeStyle.CodeStyleManager;
import com.intellij.psi.impl.source.PsiFileImpl;
import com.intellij.psi.impl.source.text.DiffLog;
import com.intellij.psi.impl.source.tree.FileElement;
import com.intellij.psi.impl.source.tree.ForeignLeafPsiElement;
import com.intellij.psi.impl.source.tree.TreeUtil;
import com.intellij.psi.text.BlockSupport;
import com.intellij.util.ExceptionUtil;
import com.intellij.util.Processor;
import com.intellij.util.SmartList;
import com.intellij.util.concurrency.BoundedTaskExecutor;
import com.intellij.util.containers.ContainerUtil;
import com.intellij.util.containers.HashSetQueue;
import com.intellij.util.ui.UIUtil;
import org.jetbrains.annotations.NonNls;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;
import org.jetbrains.annotations.TestOnly;
import org.jetbrains.ide.PooledThreadExecutor;

import javax.swing.*;
import java.text.SimpleDateFormat;
import java.util.Collections;
import java.util.Date;
import java.util.List;
import java.util.Set;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
import java.util.concurrent.locks.Lock;
import java.util.concurrent.locks.ReentrantLock;

public class DocumentCommitThread implements Runnable, Disposable, DocumentCommitProcessor {
  private static final Logger LOG = Logger.getInstance("#com.intellij.psi.impl.DocumentCommitThread");
  private static final String SYNC_COMMIT_REASON = "Sync commit";

  private final ExecutorService executor = new BoundedTaskExecutor("Document committing pool", PooledThreadExecutor.INSTANCE, 1, this);
  private final Object lock = new Object();
  private final HashSetQueue<CommitTask> documentsToCommit = new HashSetQueue<>();      // guarded by lock
  private final HashSetQueue<CommitTask> documentsToApplyInEDT = new HashSetQueue<>();  // guarded by lock
  private final ApplicationEx myApplication;
  private volatile boolean isDisposed;
  private CommitTask currentTask; // guarded by lock
  private boolean myEnabled; // true if we can do commits. set to false temporarily during the write action.  guarded by lock

  public static DocumentCommitThread getInstance() {
    return (DocumentCommitThread)ServiceManager.getService(DocumentCommitProcessor.class);
  }
  public DocumentCommitThread(final ApplicationEx application) {
    myApplication = application;
    // install listener in EDT to avoid missing events in case we are inside write action right now
    application.invokeLater(() -> {
      if (application.isDisposed()) return;
      assert !application.isWriteAccessAllowed() || application.isUnitTestMode(); // crazy stuff happens in tests, e.g. UIUtil.dispatchInvocationEvents() inside write action
      application.addApplicationListener(new ApplicationAdapter() {
        @Override
        public void beforeWriteActionStart(@NotNull Object action) {
          disable("Write action started: " + action);
        }

        @Override
        public void afterWriteActionFinished(@NotNull Object action) {
          // crazy things happen when running tests, like starting write action in one thread but firing its end in the other
          enable("Write action finished: " + action);
        }
      }, this);

      enable("Listener installed, started");
    });
  }

  @Override
  public void dispose() {
    isDisposed = true;
    synchronized (lock) {
      documentsToCommit.clear();
    }
    cancel("Stop thread");
  }

  private void disable(@NonNls @NotNull Object reason) {
    // write action has just started, all commits are useless
    synchronized (lock) {
      cancel(reason);
      myEnabled = false;
    }
    log(null, "disabled", null, reason);
  }

  private void enable(@NonNls @NotNull Object reason) {
    synchronized (lock) {
      myEnabled = true;
      wakeUpQueue();
    }
    log(null, "enabled", null, reason);
  }

  // under lock
  private void wakeUpQueue() {
    if (!isDisposed && !documentsToCommit.isEmpty()) {
      executor.execute(this);
    }
  }

  private void cancel(@NonNls @NotNull Object reason) {
    startNewTask(null, reason);
  }

  @Override
  public void commitAsynchronously(@NotNull final Project project,
                                   @NotNull final Document document,
                                   @NonNls @NotNull Object reason,
                                   @NotNull ModalityState currentModalityState) {
    assert !isDisposed : "already disposed";

    if (!project.isInitialized()) return;
    PsiFile psiFile = PsiDocumentManager.getInstance(project).getCachedPsiFile(document);
    if (psiFile == null) return;
    doQueue(project, document, getAllFileNodes(psiFile), reason, currentModalityState,
            PsiDocumentManager.getInstance(project).getLastCommittedText(document));
  }

  private void doQueue(@NotNull Project project,
                             @NotNull Document document,
                             @NotNull List<Pair<PsiFileImpl, FileASTNode>> oldFileNodes,
                             @NotNull Object reason,
                             @NotNull ModalityState currentModalityState,
                             @NotNull CharSequence lastCommittedText) {
    synchronized (lock) {
      if (!project.isInitialized()) return;  // check the project is disposed under lock.
      CommitTask newTask = createNewTaskAndCancelSimilar(project, document, oldFileNodes, reason, currentModalityState,
                                                         lastCommittedText);

      documentsToCommit.offer(newTask);
      log(project, "Queued", newTask, reason);

      wakeUpQueue();
    }
  }

  @NotNull
  private CommitTask createNewTaskAndCancelSimilar(@NotNull Project project,
                                                   @NotNull Document document,
                                                   @NotNull List<Pair<PsiFileImpl, FileASTNode>> oldFileNodes,
                                                   @NotNull Object reason,
                                                   @NotNull ModalityState currentModalityState,
                                                   @NotNull CharSequence lastCommittedText) {
    synchronized (lock) {
      for (Pair<PsiFileImpl, FileASTNode> pair : oldFileNodes) {
        assert pair.first.getProject() == project;
      }
      CommitTask newTask = new CommitTask(project, document, oldFileNodes, createProgressIndicator(), reason, currentModalityState,
                                          lastCommittedText);
      cancelAndRemoveFromDocsToCommit(newTask, reason);
      cancelAndRemoveCurrentTask(newTask, reason);
      cancelAndRemoveFromDocsToApplyInEDT(newTask, reason);

      return newTask;
    }
  }

  @SuppressWarnings({"NonConstantStringShouldBeStringBuffer", "StringConcatenationInLoop"})
  public void log(Project project, @NonNls String msg, @Nullable CommitTask task, @NonNls Object... args) {
    if (true) return;

    String indent = new SimpleDateFormat("hh:mm:ss:SSSS").format(new Date()) +
      (SwingUtilities.isEventDispatchThread() ?        "-(EDT) " :
                                                       "-(" + Thread.currentThread()+ " ");
    @NonNls
    String s = indent + msg + (task == null ? " - " : "; task: " + task);

    for (Object arg : args) {
      if (!StringUtil.isEmpty(String.valueOf(arg))) {
        s += "; "+arg;
        if (arg instanceof Document) {
          Document document = (Document)arg;
          s+= " (\""+StringUtil.first(document.getImmutableCharSequence(), 40, true).toString().replaceAll("\n", " ")+"\")";
        }
      }
    }
    if (task != null) {
      if (task.indicator.isCanceled()) {
        s += "; indicator: " + task.indicator;
      }
      Document document = task.getDocument();
      boolean stillUncommitted = !task.project.isDisposed() &&
                                 ((PsiDocumentManagerBase)PsiDocumentManager.getInstance(task.project)).isInUncommittedSet(document);
      if (stillUncommitted) {
        s += "; still uncommitted";
      }

      Set<Document> uncommitted = project == null || project.isDisposed() ? Collections.<Document>emptySet() :
                                  ((PsiDocumentManagerBase)PsiDocumentManager.getInstance(project)).myUncommittedDocuments;
      if (!uncommitted.isEmpty()) {
        s+= "; uncommitted: "+uncommitted;
      }
    }
    synchronized (lock) {
      int size = documentsToCommit.size();
      if (size != 0) {
        s += " (" + size + " documents still in queue: ";
        int i = 0;
        for (CommitTask commitTask : documentsToCommit) {
          s += commitTask + "; ";
          if (++i > 4) {
            s += " ... ";
            break;
          }
        }
        s += ")";
      }
    }

    LOG.debug(s);
  }


  // cancels all pending commits
  @TestOnly // under lock
  private void cancelAll() {
    String reason = "Cancel all in tests";
    cancel(reason);
    for (CommitTask commitTask : documentsToCommit) {
      commitTask.cancel(reason, this);
      log(commitTask.project, "Removed from background queue", commitTask);
    }
    documentsToCommit.clear();
    for (CommitTask commitTask : documentsToApplyInEDT) {
      commitTask.cancel(reason, this);
      log(commitTask.project, "Removed from EDT apply queue (sync commit called)", commitTask);
    }
    documentsToApplyInEDT.clear();
    CommitTask task = currentTask;
    if (task != null) {
      cancelAndRemoveFromDocsToCommit(task, reason);
    }
    cancel("Sync commit intervened");
    ((BoundedTaskExecutor)executor).clearAndCancelAll();
  }

  @TestOnly
  public void clearQueue() {
    synchronized (lock) {
      cancelAll();
      wakeUpQueue();
    }
  }

  private void cancelAndRemoveCurrentTask(@NotNull CommitTask newTask, @NotNull Object reason) {
    CommitTask currentTask = this.currentTask;
    if (currentTask != null && currentTask.equals(newTask)) {
      cancelAndRemoveFromDocsToCommit(currentTask, reason);
      cancel(reason);
    }
  }

  private void cancelAndRemoveFromDocsToApplyInEDT(@NotNull CommitTask newTask, @NotNull Object reason) {
    boolean removed = cancelAndRemoveFromQueue(newTask, documentsToApplyInEDT, reason);
    if (removed) {
      log(newTask.project, "Removed from EDT apply queue", newTask);
    }
  }

  private void cancelAndRemoveFromDocsToCommit(@NotNull final CommitTask newTask, @NotNull Object reason) {
    boolean removed = cancelAndRemoveFromQueue(newTask, documentsToCommit, reason);
    if (removed) {
      log(newTask.project, "Removed from background queue", newTask);
    }
  }

  private boolean cancelAndRemoveFromQueue(@NotNull CommitTask newTask, @NotNull HashSetQueue<CommitTask> queue, @NotNull Object reason) {
    CommitTask queuedTask = queue.find(newTask);
    if (queuedTask != null) {
      assert queuedTask != newTask;
      queuedTask.cancel(reason, this);
    }
    return queue.remove(newTask);
  }

  @Override
  public void run() {
    while (!isDisposed) {
      try {
        boolean polled = pollQueue();
        if (!polled) break;
      }
      catch(Throwable e) {
        LOG.error(e);
      }
    }
  }

  // returns true if queue changed
  private boolean pollQueue() {
    assert !myApplication.isDispatchThread() : Thread.currentThread();
    boolean success = false;
    Document document = null;
    Project project = null;
    CommitTask task = null;
    Object failureReason = null;
    try {
      ProgressIndicator indicator;
      synchronized (lock) {
        if (!myEnabled || (task = documentsToCommit.poll()) == null) {
          return false;
        }

        document = task.getDocument();
        indicator = task.indicator;
        project = task.project;

        if (project.isDisposed() || !((PsiDocumentManagerBase)PsiDocumentManager.getInstance(project)).isInUncommittedSet(document)) {
          log(project, "Abandon and proceed to next", task);
          return true;
        }

        if (indicator.isCanceled()) {
          return true; // document has been marked as removed, e.g. by synchronous commit
        }

        startNewTask(task, "Pulled new task");

        // transfer to documentsToApplyInEDT
        documentsToApplyInEDT.add(task);
      }

      if (indicator.isCanceled()) {
        success = false;
      }
      else {
        final CommitTask commitTask = task;
        final Ref<Pair<Runnable, Object>> result = new Ref<>();
        ProgressManager.getInstance().executeProcessUnderProgress(() -> result.set(commitUnderProgress(commitTask, false)), indicator);
        final Runnable finishRunnable = result.get().first;
        success = finishRunnable != null;
        failureReason = result.get().second;

        if (success) {
          assert !myApplication.isDispatchThread();
          TransactionGuardImpl guard = (TransactionGuardImpl)TransactionGuard.getInstance();
          guard.submitTransaction(project, guard.getModalityTransaction(task.myCreationModalityState), finishRunnable);
        }
      }
    }
    catch (ProcessCanceledException e) {
      cancel(e + " (cancel reason: "+((UserDataHolder)task.indicator).getUserData(CANCEL_REASON)+")"); // leave queue unchanged
      success = false;
      failureReason = e;
    }
    catch (Throwable e) {
      cancel(e);
      failureReason = ExceptionUtil.getThrowableText(e);
    }

    if (!success && task != null) {
      final Project finalProject = project;
      PsiDocumentManager documentManager = ReadAction.compute(() -> finalProject.isDisposed() ? null : PsiDocumentManager.getInstance(finalProject));
      if (documentManager.isUncommited(document)) { // sync commit has not intervened
        final Document finalDocument = document;
        final CharSequence[] lastCommittedText = {null};
        List<Pair<PsiFileImpl, FileASTNode>> oldFileNodes =
          ApplicationManager.getApplication().runReadAction((Computable<List<Pair<PsiFileImpl, FileASTNode>>>)() -> {
            if (finalProject.isDisposed()) return null;
            lastCommittedText[0] = PsiDocumentManager.getInstance(finalProject).getLastCommittedText(finalDocument);
            PsiFile file = documentManager.getPsiFile(finalDocument);
            return file == null ? null : getAllFileNodes(file);
          });
        if (oldFileNodes != null) {
          doQueue(project, document, oldFileNodes, "re-added on failure: " + failureReason, task.myCreationModalityState,
                  lastCommittedText[0]);
        }
      }
    }
    synchronized (lock) {
      currentTask = null; // do not cancel, it's being invokeLatered
    }

    return true;
  }

  @Override
  public void commitSynchronously(@NotNull Document document, @NotNull Project project, @NotNull PsiFile psiFile) {
    assert !isDisposed;

    if (!project.isInitialized() && !project.isDefault()) {
      @NonNls String s = project + "; Disposed: "+project.isDisposed()+"; Open: "+project.isOpen();<commit_msg>fixed NPE caused by previous commit<commit_after>/*
 * Copyright 2000-2016 JetBrains s.r.o.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package com.intellij.psi.impl;

import com.intellij.lang.ASTNode;
import com.intellij.lang.FileASTNode;
import com.intellij.openapi.Disposable;
import com.intellij.openapi.application.*;
import com.intellij.openapi.application.ex.ApplicationEx;
import com.intellij.openapi.components.ServiceManager;
import com.intellij.openapi.diagnostic.Logger;
import com.intellij.openapi.editor.Document;
import com.intellij.openapi.editor.ex.DocumentEx;
import com.intellij.openapi.progress.ProcessCanceledException;
import com.intellij.openapi.progress.ProgressIndicator;
import com.intellij.openapi.progress.ProgressManager;
import com.intellij.openapi.progress.util.StandardProgressIndicatorBase;
import com.intellij.openapi.project.Project;
import com.intellij.openapi.util.*;
import com.intellij.openapi.util.text.StringUtil;
import com.intellij.openapi.vfs.VirtualFile;
import com.intellij.pom.PomManager;
import com.intellij.pom.PomModel;
import com.intellij.pom.event.PomModelEvent;
import com.intellij.pom.impl.PomTransactionBase;
import com.intellij.pom.tree.TreeAspect;
import com.intellij.pom.tree.TreeAspectEvent;
import com.intellij.psi.*;
import com.intellij.psi.codeStyle.CodeStyleManager;
import com.intellij.psi.impl.source.PsiFileImpl;
import com.intellij.psi.impl.source.text.DiffLog;
import com.intellij.psi.impl.source.tree.FileElement;
import com.intellij.psi.impl.source.tree.ForeignLeafPsiElement;
import com.intellij.psi.impl.source.tree.TreeUtil;
import com.intellij.psi.text.BlockSupport;
import com.intellij.util.ExceptionUtil;
import com.intellij.util.Processor;
import com.intellij.util.SmartList;
import com.intellij.util.concurrency.BoundedTaskExecutor;
import com.intellij.util.containers.ContainerUtil;
import com.intellij.util.containers.HashSetQueue;
import com.intellij.util.ui.UIUtil;
import org.jetbrains.annotations.NonNls;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;
import org.jetbrains.annotations.TestOnly;
import org.jetbrains.ide.PooledThreadExecutor;

import javax.swing.*;
import java.text.SimpleDateFormat;
import java.util.Collections;
import java.util.Date;
import java.util.List;
import java.util.Set;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
import java.util.concurrent.locks.Lock;
import java.util.concurrent.locks.ReentrantLock;

public class DocumentCommitThread implements Runnable, Disposable, DocumentCommitProcessor {
  private static final Logger LOG = Logger.getInstance("#com.intellij.psi.impl.DocumentCommitThread");
  private static final String SYNC_COMMIT_REASON = "Sync commit";

  private final ExecutorService executor = new BoundedTaskExecutor("Document committing pool", PooledThreadExecutor.INSTANCE, 1, this);
  private final Object lock = new Object();
  private final HashSetQueue<CommitTask> documentsToCommit = new HashSetQueue<>();      // guarded by lock
  private final HashSetQueue<CommitTask> documentsToApplyInEDT = new HashSetQueue<>();  // guarded by lock
  private final ApplicationEx myApplication;
  private volatile boolean isDisposed;
  private CommitTask currentTask; // guarded by lock
  private boolean myEnabled; // true if we can do commits. set to false temporarily during the write action.  guarded by lock

  public static DocumentCommitThread getInstance() {
    return (DocumentCommitThread)ServiceManager.getService(DocumentCommitProcessor.class);
  }
  public DocumentCommitThread(final ApplicationEx application) {
    myApplication = application;
    // install listener in EDT to avoid missing events in case we are inside write action right now
    application.invokeLater(() -> {
      if (application.isDisposed()) return;
      assert !application.isWriteAccessAllowed() || application.isUnitTestMode(); // crazy stuff happens in tests, e.g. UIUtil.dispatchInvocationEvents() inside write action
      application.addApplicationListener(new ApplicationAdapter() {
        @Override
        public void beforeWriteActionStart(@NotNull Object action) {
          disable("Write action started: " + action);
        }

        @Override
        public void afterWriteActionFinished(@NotNull Object action) {
          // crazy things happen when running tests, like starting write action in one thread but firing its end in the other
          enable("Write action finished: " + action);
        }
      }, this);

      enable("Listener installed, started");
    });
  }

  @Override
  public void dispose() {
    isDisposed = true;
    synchronized (lock) {
      documentsToCommit.clear();
    }
    cancel("Stop thread");
  }

  private void disable(@NonNls @NotNull Object reason) {
    // write action has just started, all commits are useless
    synchronized (lock) {
      cancel(reason);
      myEnabled = false;
    }
    log(null, "disabled", null, reason);
  }

  private void enable(@NonNls @NotNull Object reason) {
    synchronized (lock) {
      myEnabled = true;
      wakeUpQueue();
    }
    log(null, "enabled", null, reason);
  }

  // under lock
  private void wakeUpQueue() {
    if (!isDisposed && !documentsToCommit.isEmpty()) {
      executor.execute(this);
    }
  }

  private void cancel(@NonNls @NotNull Object reason) {
    startNewTask(null, reason);
  }

  @Override
  public void commitAsynchronously(@NotNull final Project project,
                                   @NotNull final Document document,
                                   @NonNls @NotNull Object reason,
                                   @NotNull ModalityState currentModalityState) {
    assert !isDisposed : "already disposed";

    if (!project.isInitialized()) return;
    PsiFile psiFile = PsiDocumentManager.getInstance(project).getCachedPsiFile(document);
    if (psiFile == null) return;
    doQueue(project, document, getAllFileNodes(psiFile), reason, currentModalityState,
            PsiDocumentManager.getInstance(project).getLastCommittedText(document));
  }

  private void doQueue(@NotNull Project project,
                             @NotNull Document document,
                             @NotNull List<Pair<PsiFileImpl, FileASTNode>> oldFileNodes,
                             @NotNull Object reason,
                             @NotNull ModalityState currentModalityState,
                             @NotNull CharSequence lastCommittedText) {
    synchronized (lock) {
      if (!project.isInitialized()) return;  // check the project is disposed under lock.
      CommitTask newTask = createNewTaskAndCancelSimilar(project, document, oldFileNodes, reason, currentModalityState,
                                                         lastCommittedText);

      documentsToCommit.offer(newTask);
      log(project, "Queued", newTask, reason);

      wakeUpQueue();
    }
  }

  @NotNull
  private CommitTask createNewTaskAndCancelSimilar(@NotNull Project project,
                                                   @NotNull Document document,
                                                   @NotNull List<Pair<PsiFileImpl, FileASTNode>> oldFileNodes,
                                                   @NotNull Object reason,
                                                   @NotNull ModalityState currentModalityState,
                                                   @NotNull CharSequence lastCommittedText) {
    synchronized (lock) {
      for (Pair<PsiFileImpl, FileASTNode> pair : oldFileNodes) {
        assert pair.first.getProject() == project;
      }
      CommitTask newTask = new CommitTask(project, document, oldFileNodes, createProgressIndicator(), reason, currentModalityState,
                                          lastCommittedText);
      cancelAndRemoveFromDocsToCommit(newTask, reason);
      cancelAndRemoveCurrentTask(newTask, reason);
      cancelAndRemoveFromDocsToApplyInEDT(newTask, reason);

      return newTask;
    }
  }

  @SuppressWarnings({"NonConstantStringShouldBeStringBuffer", "StringConcatenationInLoop"})
  public void log(Project project, @NonNls String msg, @Nullable CommitTask task, @NonNls Object... args) {
    if (true) return;

    String indent = new SimpleDateFormat("hh:mm:ss:SSSS").format(new Date()) +
      (SwingUtilities.isEventDispatchThread() ?        "-(EDT) " :
                                                       "-(" + Thread.currentThread()+ " ");
    @NonNls
    String s = indent + msg + (task == null ? " - " : "; task: " + task);

    for (Object arg : args) {
      if (!StringUtil.isEmpty(String.valueOf(arg))) {
        s += "; "+arg;
        if (arg instanceof Document) {
          Document document = (Document)arg;
          s+= " (\""+StringUtil.first(document.getImmutableCharSequence(), 40, true).toString().replaceAll("\n", " ")+"\")";
        }
      }
    }
    if (task != null) {
      if (task.indicator.isCanceled()) {
        s += "; indicator: " + task.indicator;
      }
      Document document = task.getDocument();
      boolean stillUncommitted = !task.project.isDisposed() &&
                                 ((PsiDocumentManagerBase)PsiDocumentManager.getInstance(task.project)).isInUncommittedSet(document);
      if (stillUncommitted) {
        s += "; still uncommitted";
      }

      Set<Document> uncommitted = project == null || project.isDisposed() ? Collections.<Document>emptySet() :
                                  ((PsiDocumentManagerBase)PsiDocumentManager.getInstance(project)).myUncommittedDocuments;
      if (!uncommitted.isEmpty()) {
        s+= "; uncommitted: "+uncommitted;
      }
    }
    synchronized (lock) {
      int size = documentsToCommit.size();
      if (size != 0) {
        s += " (" + size + " documents still in queue: ";
        int i = 0;
        for (CommitTask commitTask : documentsToCommit) {
          s += commitTask + "; ";
          if (++i > 4) {
            s += " ... ";
            break;
          }
        }
        s += ")";
      }
    }

    LOG.debug(s);
  }


  // cancels all pending commits
  @TestOnly // under lock
  private void cancelAll() {
    String reason = "Cancel all in tests";
    cancel(reason);
    for (CommitTask commitTask : documentsToCommit) {
      commitTask.cancel(reason, this);
      log(commitTask.project, "Removed from background queue", commitTask);
    }
    documentsToCommit.clear();
    for (CommitTask commitTask : documentsToApplyInEDT) {
      commitTask.cancel(reason, this);
      log(commitTask.project, "Removed from EDT apply queue (sync commit called)", commitTask);
    }
    documentsToApplyInEDT.clear();
    CommitTask task = currentTask;
    if (task != null) {
      cancelAndRemoveFromDocsToCommit(task, reason);
    }
    cancel("Sync commit intervened");
    ((BoundedTaskExecutor)executor).clearAndCancelAll();
  }

  @TestOnly
  public void clearQueue() {
    synchronized (lock) {
      cancelAll();
      wakeUpQueue();
    }
  }

  private void cancelAndRemoveCurrentTask(@NotNull CommitTask newTask, @NotNull Object reason) {
    CommitTask currentTask = this.currentTask;
    if (currentTask != null && currentTask.equals(newTask)) {
      cancelAndRemoveFromDocsToCommit(currentTask, reason);
      cancel(reason);
    }
  }

  private void cancelAndRemoveFromDocsToApplyInEDT(@NotNull CommitTask newTask, @NotNull Object reason) {
    boolean removed = cancelAndRemoveFromQueue(newTask, documentsToApplyInEDT, reason);
    if (removed) {
      log(newTask.project, "Removed from EDT apply queue", newTask);
    }
  }

  private void cancelAndRemoveFromDocsToCommit(@NotNull final CommitTask newTask, @NotNull Object reason) {
    boolean removed = cancelAndRemoveFromQueue(newTask, documentsToCommit, reason);
    if (removed) {
      log(newTask.project, "Removed from background queue", newTask);
    }
  }

  private boolean cancelAndRemoveFromQueue(@NotNull CommitTask newTask, @NotNull HashSetQueue<CommitTask> queue, @NotNull Object reason) {
    CommitTask queuedTask = queue.find(newTask);
    if (queuedTask != null) {
      assert queuedTask != newTask;
      queuedTask.cancel(reason, this);
    }
    return queue.remove(newTask);
  }

  @Override
  public void run() {
    while (!isDisposed) {
      try {
        boolean polled = pollQueue();
        if (!polled) break;
      }
      catch(Throwable e) {
        LOG.error(e);
      }
    }
  }

  // returns true if queue changed
  private boolean pollQueue() {
    assert !myApplication.isDispatchThread() : Thread.currentThread();
    boolean success = false;
    Document document = null;
    Project project = null;
    CommitTask task = null;
    Object failureReason = null;
    try {
      ProgressIndicator indicator;
      synchronized (lock) {
        if (!myEnabled || (task = documentsToCommit.poll()) == null) {
          return false;
        }

        document = task.getDocument();
        indicator = task.indicator;
        project = task.project;

        if (project.isDisposed() || !((PsiDocumentManagerBase)PsiDocumentManager.getInstance(project)).isInUncommittedSet(document)) {
          log(project, "Abandon and proceed to next", task);
          return true;
        }

        if (indicator.isCanceled()) {
          return true; // document has been marked as removed, e.g. by synchronous commit
        }

        startNewTask(task, "Pulled new task");

        // transfer to documentsToApplyInEDT
        documentsToApplyInEDT.add(task);
      }

      if (indicator.isCanceled()) {
        success = false;
      }
      else {
        final CommitTask commitTask = task;
        final Ref<Pair<Runnable, Object>> result = new Ref<>();
        ProgressManager.getInstance().executeProcessUnderProgress(() -> result.set(commitUnderProgress(commitTask, false)), indicator);
        final Runnable finishRunnable = result.get().first;
        success = finishRunnable != null;
        failureReason = result.get().second;

        if (success) {
          assert !myApplication.isDispatchThread();
          TransactionGuardImpl guard = (TransactionGuardImpl)TransactionGuard.getInstance();
          guard.submitTransaction(project, guard.getModalityTransaction(task.myCreationModalityState), finishRunnable);
        }
      }
    }
    catch (ProcessCanceledException e) {
      cancel(e + " (cancel reason: "+((UserDataHolder)task.indicator).getUserData(CANCEL_REASON)+")"); // leave queue unchanged
      success = false;
      failureReason = e;
    }
    catch (Throwable e) {
      cancel(e);
      failureReason = ExceptionUtil.getThrowableText(e);
    }

    if (!success && task != null) {
      final Project finalProject = project;
      final Document finalDocument = document;
      Object finalFailureReason = failureReason;
      CommitTask finalTask = task;
      ReadAction.run(() -> {
        if (finalProject.isDisposed()) return;
        PsiDocumentManager documentManager = PsiDocumentManager.getInstance(finalProject);
        if (documentManager.isCommitted(finalDocument)) return; // sync commit hasn't intervened
        CharSequence lastCommittedText = documentManager.getLastCommittedText(finalDocument);
        PsiFile file = documentManager.getPsiFile(finalDocument);
        List<Pair<PsiFileImpl, FileASTNode>> oldFileNodes = file == null ? null : getAllFileNodes(file);
        if (oldFileNodes != null) {
          doQueue(finalProject, finalDocument, oldFileNodes, "re-added on failure: " + finalFailureReason,
                  finalTask.myCreationModalityState,
                  lastCommittedText);
        }
      });
    }
    synchronized (lock) {
      currentTask = null; // do not cancel, it's being invokeLatered
    }

    return true;
  }

  @Override
  public void commitSynchronously(@NotNull Document document, @NotNull Project project, @NotNull PsiFile psiFile) {
    assert !isDisposed;

    if (!project.isInitialized() && !project.isDefault()) {
      @NonNls String s = project + "; Disposed: "+project.isDisposed()+"; Open: "+project.isOpen();
{'fraction_non_alphanumeric': 0.10213287131078982, 'fraction_numerical': 0.001135141593977775, 'mean_word_length': 3.015954894433781, 'pattern_counts': {'":': 0, '<': 40, '<?xml version=': 0, '>': 49, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 42, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3021585', 'n_tokens_mistral': 9094, 'n_tokens_neox': 9030, 'n_words': 2193}