<commit_before>package de.gurkenlabs.util.io;

import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.nio.file.DirectoryStream;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.List;

public class FileUtilities {
  public static InputStream getGameResource(String file) {
    try {
      final InputStream resourceStream = ClassLoader.getSystemResourceAsStream(file);
      if (resourceStream != null) {
        return resourceStream;
      }

      final InputStream fileStream = new FileInputStream(file);
      return fileStream;
    } catch (final IOException e) {
      e.printStackTrace();
      return null;
    }
  }

  public static String getFileName(String path) {
    if (path == null || path.isEmpty()) {
      return "";
    }

    String name = path;
    final int pos = name.lastIndexOf(".");
    if (pos > 0) {
      name = name.substring(0, pos);
    }

    int lastBackslash = name.lastIndexOf("/");
    if (lastBackslash != -1) {
      name = name.substring(lastBackslash + 1, name.length());
    } else {
      int lastForwardSlash = name.lastIndexOf("\\");
      if (lastForwardSlash != -1) {
        name = name.substring(lastForwardSlash + 1, name.length());
      }
    }

    return name;
  }

  public static String getParentDirPath(String fileOrDirPath) {
    if (fileOrDirPath.contains(File.separator)) {
      return fileOrDirPath.substring(0, fileOrDirPath.lastIndexOf(File.separatorChar, fileOrDirPath.length()));
    } else if (fileOrDirPath.contains("/")) {
      return fileOrDirPath.substring(0, fileOrDirPath.lastIndexOf("/") + 1);
    }

    return "";
  }

  public static String getExtension(String fileName) {
    try {
      return fileName.substring(fileName.lastIndexOf(".") + 1);
    } catch (Exception e) {
      return "";
    }
  }

  public static boolean deleteDir(File dir) {
    if (dir.isDirectory()) {
      String[] children = dir.list();
      for (int i = 0; i < children.length; i++) {
        boolean success = deleteDir(new File(dir, children[i]));
        if (!success) {
          return false;
        }
      }
    }

    return dir.delete(); // The directory is empty now and can be deleted.
  }
  
  public static List<String> findFiles(List<String> fileNames, Path dir, String extension) {
    final String[] blackList = new String[] { "\\bin", "\\screenshots" };
    try (DirectoryStream<Path> stream = Files.newDirectoryStream(dir)) {
      for (Path path : stream) {
        if (path.toFile().isDirectory()) {<commit_msg>Implemented command line file loading.<commit_after>package de.gurkenlabs.util.io;

import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.nio.file.DirectoryStream;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.List;

public class FileUtilities {
  public static InputStream getGameResource(String file) {
    try {
      final InputStream resourceStream = ClassLoader.getSystemResourceAsStream(file);
      if (resourceStream != null) {
        return resourceStream;
      }

      final InputStream fileStream = new FileInputStream(file);
      return fileStream;
    } catch (final IOException e) {
      e.printStackTrace();
      return null;
    }
  }

  public static String getFileName(String path) {
    if (path == null || path.isEmpty()) {
      return "";
    }

    String name = path;
    final int pos = name.lastIndexOf(".");
    if (pos > 0) {
      name = name.substring(0, pos);
    }

    int lastBackslash = name.lastIndexOf("/");
    if (lastBackslash != -1) {
      name = name.substring(lastBackslash + 1, name.length());
    } else {
      int lastForwardSlash = name.lastIndexOf("\\");
      if (lastForwardSlash != -1) {
        name = name.substring(lastForwardSlash + 1, name.length());
      }
    }

    return name;
  }

  public static String getParentDirPath(String fileOrDirPath) {
    if (fileOrDirPath.contains(File.separator)) {
      return fileOrDirPath.substring(0, fileOrDirPath.lastIndexOf(File.separatorChar, fileOrDirPath.length()));
    } else if (fileOrDirPath.contains("/")) {
      return fileOrDirPath.substring(0, fileOrDirPath.lastIndexOf("/") + 1);
    }

    return "";
  }

  public static String getExtension(File file) {
    return getExtension(file.getAbsolutePath());
  }

  public static String getExtension(String fileName) {
    try {
      return fileName.substring(fileName.lastIndexOf(".") + 1);
    } catch (Exception e) {
      return "";
    }
  }

  public static boolean deleteDir(File dir) {
    if (dir.isDirectory()) {
      String[] children = dir.list();
      for (int i = 0; i < children.length; i++) {
        boolean success = deleteDir(new File(dir, children[i]));
        if (!success) {
          return false;
        }
      }
    }

    return dir.delete(); // The directory is empty now and can be deleted.
  }

  public static List<String> findFiles(List<String> fileNames, Path dir, String extension) {
    final String[] blackList = new String[] { "\\bin", "\\screenshots" };
    try (DirectoryStream<Path> stream = Files.newDirectoryStream(dir)) {
      for (Path path : stream) {
        if (path.toFile().isDirectory()) {
{'fraction_non_alphanumeric': 0.1280741581536133, 'fraction_numerical': 0.004161937192584185, 'mean_word_length': 2.942580164056674, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1450316', 'n_tokens_mistral': 1593, 'n_tokens_neox': 1556, 'n_words': 393}