<commit_before>package com.intellij.openapi.roots.impl;

import com.intellij.ide.startup.StartupManagerEx;
import com.intellij.openapi.components.ProjectComponent;
import com.intellij.openapi.diagnostic.Logger;
import com.intellij.openapi.fileTypes.FileTypeEvent;
import com.intellij.openapi.fileTypes.FileTypeListener;
import com.intellij.openapi.fileTypes.FileTypeManager;
import com.intellij.openapi.module.Module;
import com.intellij.openapi.module.ModuleManager;
import com.intellij.openapi.progress.ProgressIndicator;
import com.intellij.openapi.progress.ProgressManager;
import com.intellij.openapi.project.Project;
import com.intellij.openapi.project.ProjectBundle;
import com.intellij.openapi.roots.*;
import com.intellij.openapi.util.Key;
import com.intellij.openapi.vfs.*;
import com.intellij.psi.PsiManager;
import com.intellij.psi.PsiNameHelper;
import com.intellij.psi.impl.PsiManagerConfiguration;
import com.intellij.util.ArrayUtil;
import com.intellij.util.EventDispatcher;
import gnu.trove.THashMap;
import junit.framework.Assert;

import java.util.*;

public class DirectoryIndexImpl extends DirectoryIndex implements ProjectComponent {
  private static final Logger LOG = Logger.getInstance("#com.intellij.openapi.roots.impl.DirectoryIndexImpl");

  private boolean LAZY_MODE;

  private final Project myProject;

  private boolean myInitialized = false;
  private boolean myDisposed = false;

  private Map<VirtualFile, DirectoryInfo> myDirToInfoMap = new THashMap<VirtualFile, DirectoryInfo>();
  private Map<String, VirtualFile[]> myPackageNameToDirsMap = new THashMap<String, VirtualFile[]>();

  private VirtualFileListener myVirtualFileListener;
  private FileTypeListener myFileTypeListener;
  private ModuleRootListener myRootListener;

  private PsiNameHelper myNameHelper;

  public DirectoryIndexImpl(Project project, PsiManagerConfiguration psiManagerConfiguration, StartupManagerEx startupManagerEx) {
    myProject = project;

    LAZY_MODE = !psiManagerConfiguration.REPOSITORY_ENABLED;
    startupManagerEx.registerPreStartupActivity(
      new Runnable() {
        public void run() {
          initialize();
        }
      }
    );
  }

  public String getComponentName() {
    return "DirectoryIndex";
  }

  public void initComponent() {
  }

  public void disposeComponent() {
    if (myInitialized) {
      VirtualFileManager.getInstance().removeVirtualFileListener(myVirtualFileListener);
      FileTypeManager.getInstance().removeFileTypeListener(myFileTypeListener);
      ProjectRootManager.getInstance(myProject).removeModuleRootListener(myRootListener);
    }
    myDisposed = true;
  }

  public void projectOpened() {
  }

  public void projectClosed() {
  }

  // for tests
  public void checkConsistency() {
    checkConsistency(false);
    checkConsistency(true);
  }

  private void checkConsistency(boolean reverseAllSets) {
    Assert.assertTrue(myInitialized);
    Assert.assertTrue(!myDisposed);

    Map<VirtualFile, DirectoryInfo> oldDirToInfoMap = myDirToInfoMap;
    myDirToInfoMap = new THashMap<VirtualFile, DirectoryInfo>();

    Map<String, VirtualFile[]> oldPackageNameToDirsMap = myPackageNameToDirsMap;
    myPackageNameToDirsMap = new THashMap<String, VirtualFile[]>();

    _initialize(reverseAllSets, null);

    if (LAZY_MODE) {
      Map<VirtualFile, DirectoryInfo> newDirToInfoMap = myDirToInfoMap;
      Map<String, VirtualFile[]> newPackageNameToDirsMap = myPackageNameToDirsMap;
      myDirToInfoMap = oldDirToInfoMap;
      myPackageNameToDirsMap = oldPackageNameToDirsMap;

      Set<VirtualFile> allDirsSet = newDirToInfoMap.keySet();
      for (VirtualFile dir : allDirsSet) {
        getInfoForDirectory(dir);
      }

      myDirToInfoMap = newDirToInfoMap;
      myPackageNameToDirsMap = newPackageNameToDirsMap;
    }

    Set<VirtualFile> keySet = myDirToInfoMap.keySet();
    Assert.assertEquals(keySet, oldDirToInfoMap.keySet());
    for (VirtualFile file : keySet) {
      DirectoryInfo info1 = myDirToInfoMap.get(file);
      DirectoryInfo info2 = oldDirToInfoMap.get(file);
      Assert.assertEquals(info1, info2);
    }

    Assert.assertEquals(myPackageNameToDirsMap.keySet(), oldPackageNameToDirsMap.keySet());
    for (Map.Entry<String, VirtualFile[]> entry : myPackageNameToDirsMap.entrySet()) {
      String packageName = entry.getKey();
      VirtualFile[] dirs = entry.getValue();
      VirtualFile[] dirs1 = oldPackageNameToDirsMap.get(packageName);

      HashSet<VirtualFile> set1 = new HashSet<VirtualFile>();
      set1.addAll(Arrays.asList(dirs));
      HashSet<VirtualFile> set2 = new HashSet<VirtualFile>();
      set2.addAll(Arrays.asList(dirs1));
      Assert.assertEquals(set1, set2);
    }
  }

  /**
   * @fabrique Used in fabrique
   */
  public void initialize() {<commit_msg>Fixed test broken by cdr.<commit_after>package com.intellij.openapi.roots.impl;

import com.intellij.ide.startup.StartupManagerEx;
import com.intellij.openapi.components.ProjectComponent;
import com.intellij.openapi.diagnostic.Logger;
import com.intellij.openapi.fileTypes.FileTypeEvent;
import com.intellij.openapi.fileTypes.FileTypeListener;
import com.intellij.openapi.fileTypes.FileTypeManager;
import com.intellij.openapi.module.Module;
import com.intellij.openapi.module.ModuleManager;
import com.intellij.openapi.progress.ProgressIndicator;
import com.intellij.openapi.progress.ProgressManager;
import com.intellij.openapi.project.Project;
import com.intellij.openapi.project.ProjectBundle;
import com.intellij.openapi.roots.*;
import com.intellij.openapi.util.Key;
import com.intellij.openapi.vfs.*;
import com.intellij.psi.PsiManager;
import com.intellij.psi.PsiNameHelper;
import com.intellij.psi.impl.PsiManagerConfiguration;
import com.intellij.util.ArrayUtil;
import com.intellij.util.EventDispatcher;
import gnu.trove.THashMap;
import junit.framework.Assert;

import java.util.*;

public class DirectoryIndexImpl extends DirectoryIndex implements ProjectComponent {
  private static final Logger LOG = Logger.getInstance("#com.intellij.openapi.roots.impl.DirectoryIndexImpl");

  private boolean LAZY_MODE;

  private final Project myProject;

  private boolean myInitialized = false;
  private boolean myDisposed = false;

  private Map<VirtualFile, DirectoryInfo> myDirToInfoMap = new THashMap<VirtualFile, DirectoryInfo>();
  private Map<String, VirtualFile[]> myPackageNameToDirsMap = new THashMap<String, VirtualFile[]>();

  private VirtualFileListener myVirtualFileListener;
  private FileTypeListener myFileTypeListener;
  private ModuleRootListener myRootListener;

  private PsiNameHelper myNameHelper;

  public DirectoryIndexImpl(Project project, PsiManagerConfiguration psiManagerConfiguration, StartupManagerEx startupManagerEx) {
    myProject = project;

    LAZY_MODE = !psiManagerConfiguration.REPOSITORY_ENABLED;
    startupManagerEx.registerPreStartupActivity(
      new Runnable() {
        public void run() {
          initialize();
        }
      }
    );
  }

  public String getComponentName() {
    return "DirectoryIndex";
  }

  public void initComponent() {
  }

  public void disposeComponent() {
    if (myInitialized) {
      VirtualFileManager.getInstance().removeVirtualFileListener(myVirtualFileListener);
      FileTypeManager.getInstance().removeFileTypeListener(myFileTypeListener);
      ProjectRootManager.getInstance(myProject).removeModuleRootListener(myRootListener);
    }
    myDisposed = true;
  }

  public void projectOpened() {
  }

  public void projectClosed() {
  }

  // for tests
  public void checkConsistency() {
    checkConsistency(false);
    checkConsistency(true);
  }

  private void checkConsistency(boolean reverseAllSets) {
    Assert.assertTrue(myInitialized);
    Assert.assertTrue(!myDisposed);

    Map<VirtualFile, DirectoryInfo> oldDirToInfoMap = myDirToInfoMap;
    myDirToInfoMap = new THashMap<VirtualFile, DirectoryInfo>();

    Map<String, VirtualFile[]> oldPackageNameToDirsMap = myPackageNameToDirsMap;
    myPackageNameToDirsMap = new THashMap<String, VirtualFile[]>();

    _initialize(reverseAllSets, null);

    if (LAZY_MODE) {
      Map<VirtualFile, DirectoryInfo> newDirToInfoMap = myDirToInfoMap;
      Map<String, VirtualFile[]> newPackageNameToDirsMap = myPackageNameToDirsMap;
      myDirToInfoMap = oldDirToInfoMap;
      myPackageNameToDirsMap = oldPackageNameToDirsMap;

      Set<VirtualFile> allDirsSet = newDirToInfoMap.keySet();
      for (VirtualFile dir : allDirsSet) {
        getInfoForDirectory(dir);
      }

      myDirToInfoMap = newDirToInfoMap;
      myPackageNameToDirsMap = newPackageNameToDirsMap;
    }

    Set<VirtualFile> keySet = myDirToInfoMap.keySet();
    Assert.assertEquals(keySet.size(), oldDirToInfoMap.keySet().size());
    for (VirtualFile file : keySet) {
      DirectoryInfo info1 = myDirToInfoMap.get(file);
      DirectoryInfo info2 = oldDirToInfoMap.get(file);
      Assert.assertEquals(info1, info2);
    }

    Assert.assertEquals(myPackageNameToDirsMap.keySet().size(), oldPackageNameToDirsMap.keySet().size());
    for (Map.Entry<String, VirtualFile[]> entry : myPackageNameToDirsMap.entrySet()) {
      String packageName = entry.getKey();
      VirtualFile[] dirs = entry.getValue();
      VirtualFile[] dirs1 = oldPackageNameToDirsMap.get(packageName);

      HashSet<VirtualFile> set1 = new HashSet<VirtualFile>();
      set1.addAll(Arrays.asList(dirs));
      HashSet<VirtualFile> set2 = new HashSet<VirtualFile>();
      set2.addAll(Arrays.asList(dirs1));
      Assert.assertEquals(set1, set2);
    }
  }

  /**
   * @fabrique Used in fabrique
   */
  public void initialize() {
{'fraction_non_alphanumeric': 0.09715295095594348, 'fraction_numerical': 0.0024937655860349127, 'mean_word_length': 4.7842548076923075, 'pattern_counts': {'":': 0, '<': 37, '<?xml version=': 0, '>': 37, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5216218', 'n_tokens_mistral': 2954, 'n_tokens_neox': 2877, 'n_words': 510}