<commit_before>/*
 * Copyright (c) 2000-2005 by JetBrains s.r.o. All Rights Reserved.
 * Use is subject to license terms.
 */
package com.intellij.util.indexing;

import com.intellij.openapi.roots.ContentIterator;
import com.intellij.openapi.vfs.VfsUtil;
import com.intellij.openapi.vfs.VirtualFile;
import com.intellij.openapi.vfs.VirtualFileManager;
import com.intellij.util.containers.ContainerUtil;
import gnu.trove.THashSet;

import java.util.Set;

/**
 * @author peter
 */
public class AdditionalIndexableFileSet implements IndexableFileSet {
  private final Set<VirtualFile> myRoots = new THashSet<VirtualFile>();

  public AdditionalIndexableFileSet() {
    for (IndexedRootsProvider provider : IndexedRootsProvider.EP_NAME.getExtensions()) {
      for (String url : provider.getRootsToIndex()) {
        ContainerUtil.addIfNotNull(VirtualFileManager.getInstance().findFileByUrl(url), myRoots);
      }
    }
  }

  public boolean isInSet(VirtualFile file) {
    for (final VirtualFile root : myRoots) {
      if (VfsUtil.isAncestor(root, file, false)) {
        return true;
      }
    }
    return false;
  }

  public void iterateIndexableFilesIn(VirtualFile file, ContentIterator iterator) {
    iterator.processFile(file);
    if (file.isDirectory()) {
      for (VirtualFile virtualFile : file.getChildren()) {
        iterateIndexableFilesIn(virtualFile, iterator);
      }
    }
  }
}
<commit_msg>fix incorrect indexable files iteration<commit_after>/*
 * Copyright (c) 2000-2005 by JetBrains s.r.o. All Rights Reserved.
 * Use is subject to license terms.
 */
package com.intellij.util.indexing;

import com.intellij.openapi.roots.ContentIterator;
import com.intellij.openapi.vfs.VfsUtil;
import com.intellij.openapi.vfs.VirtualFile;
import com.intellij.openapi.vfs.VirtualFileManager;
import com.intellij.util.containers.ContainerUtil;
import gnu.trove.THashSet;

import java.util.Set;

/**
 * @author peter
 */
public class AdditionalIndexableFileSet implements IndexableFileSet {
  private final Set<VirtualFile> myRoots = new THashSet<VirtualFile>();

  public AdditionalIndexableFileSet() {
    for (IndexedRootsProvider provider : IndexedRootsProvider.EP_NAME.getExtensions()) {
      for (String url : provider.getRootsToIndex()) {
        ContainerUtil.addIfNotNull(VirtualFileManager.getInstance().findFileByUrl(url), myRoots);
      }
    }
  }

  public boolean isInSet(VirtualFile file) {
    for (final VirtualFile root : myRoots) {
      if (VfsUtil.isAncestor(root, file, false)) {
        return true;
      }
    }
    return false;
  }

  public void iterateIndexableFilesIn(VirtualFile file, ContentIterator iterator) {
    if (!isInSet(file)) return;

    if (file.isDirectory()) {
      for (VirtualFile child : file.getChildren()) {
        iterateIndexableFilesIn(child, iterator);
      }
    }
    else {
      iterator.processFile(file);
    }
  }
}

{'fraction_non_alphanumeric': 0.10573600552868002, 'fraction_numerical': 0.0055286800276434, 'mean_word_length': 3.8737373737373737, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4265833', 'n_tokens_mistral': 932, 'n_tokens_neox': 902, 'n_words': 186}