<commit_before>package org.jetbrains.jps.javac;

import com.intellij.openapi.util.io.FileUtil;
import com.intellij.openapi.util.text.StringUtil;
import com.sun.tools.javac.util.*;
import com.sun.tools.javac.util.List;

import javax.lang.model.SourceVersion;
import javax.tools.FileObject;
import javax.tools.JavaFileObject;
import java.io.*;
import java.lang.ref.SoftReference;
import java.lang.reflect.Field;
import java.net.URI;
import java.net.URISyntaxException;
import java.nio.ByteBuffer;
import java.nio.CharBuffer;
import java.nio.charset.*;
import java.util.*;
import java.util.concurrent.ConcurrentHashMap;

/**
 * WARNING: Loaded via reflection, do not delete
 *
 * @author nik
 * @noinspection UnusedDeclaration
 */
class OptimizedFileManager extends DefaultFileManager {
  private boolean myUseZipFileIndex;
  private final Map<File, Archive> myArchives;
  private final Map<File, Boolean> myIsFile = new ConcurrentHashMap<File, Boolean>();
  private final Map<InputFileObject, SoftReference<CharBuffer>> myContentCache = new HashMap<InputFileObject, SoftReference<CharBuffer>>();

  public OptimizedFileManager() throws Throwable {
    super(new Context(), true, null);
    final Field archivesField = DefaultFileManager.class.getDeclaredField("archives");
    archivesField.setAccessible(true);
    myArchives = (Map<File, Archive>) archivesField.get(this);

    try {
      final Field useZipFileIndexField = DefaultFileManager.class.getDeclaredField("useZipFileIndex");
      useZipFileIndexField.setAccessible(true);
      myUseZipFileIndex = (Boolean) useZipFileIndexField.get(this);
    }
    catch (Exception e) {
      myUseZipFileIndex = false;
    }
  }

  @Override
  public FileObject getFileForInput(Location location, String packageName, String relativeName) throws IOException {
    final String name = StringUtil.isEmpty(packageName) ? FileUtil.toSystemIndependentName(relativeName) : (packageName.replace('.', '/') + "/" + FileUtil.toSystemIndependentName(relativeName));
    return getFileForInput(location, name);
  }

  @Override
  public JavaFileObject getJavaFileForInput(Location location, String className, JavaFileObject.Kind kind) throws IOException {
    final String name = className.replace('.', '/') + kind.extension;
    return getFileForInput(location, name);
  }

  @Override
  public Iterable<? extends JavaFileObject> getJavaFileObjectsFromFiles(Iterable<? extends File> files) {
    java.util.List<InputFileObject> result;
    if (files instanceof Collection) {
      result = new ArrayList<InputFileObject>(((Collection)files).size());
    }
    else {
      result = new ArrayList<InputFileObject>();
    }
    for (File f: files) {
      result.add(new InputFileObject(f));
    }
    return result;
  }

  @Override
  public Iterable<JavaFileObject> list(Location location, String packageName, Set<JavaFileObject.Kind> kinds, boolean recurse) throws IOException {
    Iterable<? extends File> path = getLocation(location);
    if (path == null) return Collections.emptyList();

    String relativePath = packageName.replace('.', File.separatorChar);
    ListBuffer<JavaFileObject> results = new ListBuffer<JavaFileObject>();

    for (File root : path) {
      Archive archive = myArchives.get(root);
      final boolean isFile;
      if (archive != null) {
        isFile = true;
      }
      else {
        isFile = isFile(root);
      }
      if (isFile) {
        collectFromArchive(root, archive, relativePath, kinds, recurse, results);
      }
      else {
        File directory = relativePath.length() != 0 ? new File(root, relativePath) : root;
        if (recurse) {
          collectFromDirectoryRecursively(directory, kinds, results, true);
        }
        else {
          collectFromDirectory(directory, kinds, false, results);
        }
      }
    }

    return results.toList();
  }

  private boolean isFile(File root) {
    Boolean cachedIsFile = myIsFile.get(root);
    if (cachedIsFile == null) {
      cachedIsFile = Boolean.valueOf(root.isFile());
      myIsFile.put(root, cachedIsFile);
    }
    return cachedIsFile.booleanValue();
  }

  private void collectFromArchive(File root, Archive archive, String relativePath, Set<JavaFileObject.Kind> kinds, boolean recurse, ListBuffer<JavaFileObject> result) {
    if (archive == null) {
      try {
        archive = openArchive(root);
      }
      catch (IOException ex) {
        log.error("error.reading.file", root, ex.getLocalizedMessage());
        return;
      }
    }
    final String separator = myUseZipFileIndex ? File.separator : "/";
    if (relativePath.length() != 0) {
      if (!myUseZipFileIndex) {
        relativePath = relativePath.replace('\\', '/');
      }
      if (!relativePath.endsWith(separator)) {
        relativePath = relativePath + separator;
      }
    }

    collectArchiveFiles(archive, relativePath, kinds, result);
    if (recurse) {
      for (String s : archive.getSubdirectories()) {
        if (s.startsWith(relativePath) && !s.equals(relativePath)) {
          if (!s.endsWith(separator)) {
            s += separator;
          }
          collectArchiveFiles(archive, s, kinds, result);
        }
      }
    }
  }

  private void collectFromDirectory(File directory, Set<JavaFileObject.Kind> fileKinds, boolean recurse, ListBuffer<JavaFileObject> result) {
    final File[] children = directory.listFiles();
    if (children != null) {
      final boolean acceptUnknownFiles = fileKinds.contains(JavaFileObject.Kind.OTHER);
      for (File child : children) {
        if (isValidFile(child.getName(), fileKinds)) {
          if (acceptUnknownFiles && !isFile(child)) {
            continue;
          }
          final JavaFileObject fe = new InputFileObject(child);
          result.append(fe);
        }
      }
    }
  }

  private void collectFromDirectoryRecursively(File file, Set<JavaFileObject.Kind> fileKinds, ListBuffer<JavaFileObject> result, boolean isRootCall) {
    final File[] children = file.listFiles();
    final String name = file.getName();
    if (children != null) { // is directory
      if (isRootCall || SourceVersion.isIdentifier(name)) {
        for (File child : children) {
          collectFromDirectoryRecursively(child, fileKinds, result, false);
        }
      }
    }
    else {
      if (isValidFile(name, fileKinds)) {<commit_msg>remove unused param<commit_after>package org.jetbrains.jps.javac;

import com.intellij.openapi.util.io.FileUtil;
import com.intellij.openapi.util.text.StringUtil;
import com.sun.tools.javac.util.*;
import com.sun.tools.javac.util.List;

import javax.lang.model.SourceVersion;
import javax.tools.FileObject;
import javax.tools.JavaFileObject;
import java.io.*;
import java.lang.ref.SoftReference;
import java.lang.reflect.Field;
import java.net.URI;
import java.net.URISyntaxException;
import java.nio.ByteBuffer;
import java.nio.CharBuffer;
import java.nio.charset.*;
import java.util.*;
import java.util.concurrent.ConcurrentHashMap;

/**
 * WARNING: Loaded via reflection, do not delete
 *
 * @author nik
 * @noinspection UnusedDeclaration
 */
class OptimizedFileManager extends DefaultFileManager {
  private boolean myUseZipFileIndex;
  private final Map<File, Archive> myArchives;
  private final Map<File, Boolean> myIsFile = new ConcurrentHashMap<File, Boolean>();
  private final Map<InputFileObject, SoftReference<CharBuffer>> myContentCache = new HashMap<InputFileObject, SoftReference<CharBuffer>>();

  public OptimizedFileManager() throws Throwable {
    super(new Context(), true, null);
    final Field archivesField = DefaultFileManager.class.getDeclaredField("archives");
    archivesField.setAccessible(true);
    myArchives = (Map<File, Archive>) archivesField.get(this);

    try {
      final Field useZipFileIndexField = DefaultFileManager.class.getDeclaredField("useZipFileIndex");
      useZipFileIndexField.setAccessible(true);
      myUseZipFileIndex = (Boolean) useZipFileIndexField.get(this);
    }
    catch (Exception e) {
      myUseZipFileIndex = false;
    }
  }

  @Override
  public FileObject getFileForInput(Location location, String packageName, String relativeName) throws IOException {
    final String name = StringUtil.isEmpty(packageName) ? FileUtil.toSystemIndependentName(relativeName) : (packageName.replace('.', '/') + "/" + FileUtil.toSystemIndependentName(relativeName));
    return getFileForInput(location, name);
  }

  @Override
  public JavaFileObject getJavaFileForInput(Location location, String className, JavaFileObject.Kind kind) throws IOException {
    final String name = className.replace('.', '/') + kind.extension;
    return getFileForInput(location, name);
  }

  @Override
  public Iterable<? extends JavaFileObject> getJavaFileObjectsFromFiles(Iterable<? extends File> files) {
    java.util.List<InputFileObject> result;
    if (files instanceof Collection) {
      result = new ArrayList<InputFileObject>(((Collection)files).size());
    }
    else {
      result = new ArrayList<InputFileObject>();
    }
    for (File f: files) {
      result.add(new InputFileObject(f));
    }
    return result;
  }

  @Override
  public Iterable<JavaFileObject> list(Location location, String packageName, Set<JavaFileObject.Kind> kinds, boolean recurse) throws IOException {
    Iterable<? extends File> path = getLocation(location);
    if (path == null) return Collections.emptyList();

    String relativePath = packageName.replace('.', File.separatorChar);
    ListBuffer<JavaFileObject> results = new ListBuffer<JavaFileObject>();

    for (File root : path) {
      Archive archive = myArchives.get(root);
      final boolean isFile;
      if (archive != null) {
        isFile = true;
      }
      else {
        isFile = isFile(root);
      }
      if (isFile) {
        collectFromArchive(root, archive, relativePath, kinds, recurse, results);
      }
      else {
        File directory = relativePath.length() != 0 ? new File(root, relativePath) : root;
        if (recurse) {
          collectFromDirectoryRecursively(directory, kinds, results, true);
        }
        else {
          collectFromDirectory(directory, kinds, results);
        }
      }
    }

    return results.toList();
  }

  private boolean isFile(File root) {
    Boolean cachedIsFile = myIsFile.get(root);
    if (cachedIsFile == null) {
      cachedIsFile = Boolean.valueOf(root.isFile());
      myIsFile.put(root, cachedIsFile);
    }
    return cachedIsFile.booleanValue();
  }

  private void collectFromArchive(File root, Archive archive, String relativePath, Set<JavaFileObject.Kind> kinds, boolean recurse, ListBuffer<JavaFileObject> result) {
    if (archive == null) {
      try {
        archive = openArchive(root);
      }
      catch (IOException ex) {
        log.error("error.reading.file", root, ex.getLocalizedMessage());
        return;
      }
    }
    final String separator = myUseZipFileIndex ? File.separator : "/";
    if (relativePath.length() != 0) {
      if (!myUseZipFileIndex) {
        relativePath = relativePath.replace('\\', '/');
      }
      if (!relativePath.endsWith(separator)) {
        relativePath = relativePath + separator;
      }
    }

    collectArchiveFiles(archive, relativePath, kinds, result);
    if (recurse) {
      for (String s : archive.getSubdirectories()) {
        if (s.startsWith(relativePath) && !s.equals(relativePath)) {
          if (!s.endsWith(separator)) {
            s += separator;
          }
          collectArchiveFiles(archive, s, kinds, result);
        }
      }
    }
  }

  private void collectFromDirectory(File directory, Set<JavaFileObject.Kind> fileKinds, ListBuffer<JavaFileObject> result) {
    final File[] children = directory.listFiles();
    if (children != null) {
      final boolean acceptUnknownFiles = fileKinds.contains(JavaFileObject.Kind.OTHER);
      for (File child : children) {
        if (isValidFile(child.getName(), fileKinds)) {
          if (acceptUnknownFiles && !isFile(child)) {
            continue;
          }
          final JavaFileObject fe = new InputFileObject(child);
          result.append(fe);
        }
      }
    }
  }

  private void collectFromDirectoryRecursively(File file, Set<JavaFileObject.Kind> fileKinds, ListBuffer<JavaFileObject> result, boolean isRootCall) {
    final File[] children = file.listFiles();
    final String name = file.getName();
    if (children != null) { // is directory
      if (isRootCall || SourceVersion.isIdentifier(name)) {
        for (File child : children) {
          collectFromDirectoryRecursively(child, fileKinds, result, false);
        }
      }
    }
    else {
      if (isValidFile(name, fileKinds)) {
{'fraction_non_alphanumeric': 0.11253156565656566, 'fraction_numerical': 0.0003156565656565657, 'mean_word_length': 3.4404344779257183, 'pattern_counts': {'":': 0, '<': 51, '<?xml version=': 0, '>': 51, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4155167', 'n_tokens_mistral': 3659, 'n_tokens_neox': 3570, 'n_words': 823}