<commit_before>package fitnesse.updates;

import util.FileUtil;

import java.io.File;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;

public class UpdateFileList {
  private List<String> mainDirectories;
  private String updateListContent;
  private String updateDoNotCopyOverContent;
  private HashSet<String> doNotReplaceFiles = new HashSet<String>();
  private String baseDirectory = "";
  static UpdateFileList testUpdater = null;

  public static void main(String[] args) {
    UpdateFileList updater = testUpdater != null ? testUpdater : new UpdateFileList();

    updater.parseCommandLine(args);
    if (updater.directoriesAreValid()) {
      updater.createUpdateList();
      updater.createDoNotUpdateList();
    } else {
      updater.printMessage("Some directories are invalid.");
      updater.exit();
    }
  }

  void printMessage(String message) {
    System.err.println(message);
  }

  void exit() {
    System.exit(1);
  }

  public UpdateFileList() {
    mainDirectories = new ArrayList<String>();
    updateListContent = "";
    updateDoNotCopyOverContent = "";
  }

  public boolean parseCommandLine(String[] args) {
    if (args.length == 0)
      return false;
    for (String arg : args)
      parseArgument(arg);
    return true;
  }

  private void parseArgument(String arg) {
    if (arg.startsWith("-doNotReplace:"))
      addADoNotReplaceFileName(arg);
    else if (arg.startsWith("-baseDirectory:"))
      parseBaseDirectory(arg);
    else
      mainDirectories.add(baseDirectory + arg);
  }

  private void addADoNotReplaceFileName(String arg) {
    String[] components = arg.split(":");
    doNotReplaceFiles.add(components[1]);
  }

  private void parseBaseDirectory(String arg) {
    baseDirectory = arg.substring(arg.indexOf(':')+1);
    if (!baseDirectory.endsWith("/"))
      baseDirectory += "/";
  }

  public List<String> getDirectories() {
    return mainDirectories;
  }

  public boolean directoriesAreValid() {
    for (String dirName : mainDirectories) {
      File checkFile = new File(dirName);
      if (!checkFile.exists()) {
        return false;
      }
    }
    return true;
  }

  public File createUpdateList() {
    for (String dirName : mainDirectories)
      addFilePathsToList(dirName);

    return FileUtil.createFile(new File("updateList"), updateListContent);

  }

  private void addFilePathsToList(String directoryPath) {
    File directory = new File(directoryPath);
    File[] files = FileUtil.getDirectoryListing(directory);
    for (File childFile : files)
      addFilePathToAppropriateList(directoryPath, childFile);

  }

  private void addFilePathToAppropriateList(String directoryPath, File childFile) {
    String childPath = directoryPath + "/" + childFile.getName();
    if (childFile.isDirectory())
      addFilePathsToList(childPath);
    else if (isDoNotReplaceFile(childFile))
      updateDoNotCopyOverContent += makePathLine(childPath);
    else
      updateListContent += makePathLine(childPath);
  }

  private String makePathLine(String path) {
    if (baseDirectory != null && baseDirectory.length() > 0 && path.startsWith(baseDirectory))
      path = path.replace(baseDirectory, "");
    if (path.startsWith("/"))
      path = path.substring(1);
    return path + "\n";
  }

  private boolean isDoNotReplaceFile(File file) {
    String name = file.getPath();
    String baseDirectoryOnOS = baseDirectory.replace( "/", File.separator);
    name = name.replace(baseDirectoryOnOS, "");
    name = name.replace(File.separator, "/");
    return doNotReplaceFiles.contains(name);
  }

  public File createDoNotUpdateList() {
    if (updateDoNotCopyOverContent.equals(""))
      for (String dirName : mainDirectories)
        addFilePathsToList(dirName);
<commit_msg>Do not add .zip (backup) files to updateList<commit_after>package fitnesse.updates;

import util.FileUtil;

import java.io.File;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;

public class UpdateFileList {
  private List<String> mainDirectories;
  private String updateListContent;
  private String updateDoNotCopyOverContent;
  private HashSet<String> doNotReplaceFiles = new HashSet<String>();
  private String baseDirectory = "";
  static UpdateFileList testUpdater = null;

  public static void main(String[] args) {
    UpdateFileList updater = testUpdater != null ? testUpdater : new UpdateFileList();

    updater.parseCommandLine(args);
    if (updater.directoriesAreValid()) {
      updater.createUpdateList();
      updater.createDoNotUpdateList();
    } else {
      updater.printMessage("Some directories are invalid.");
      updater.exit();
    }
  }

  void printMessage(String message) {
    System.err.println(message);
  }

  void exit() {
    System.exit(1);
  }

  public UpdateFileList() {
    mainDirectories = new ArrayList<String>();
    updateListContent = "";
    updateDoNotCopyOverContent = "";
  }

  public boolean parseCommandLine(String[] args) {
    if (args.length == 0)
      return false;
    for (String arg : args)
      parseArgument(arg);
    return true;
  }

  private void parseArgument(String arg) {
    if (arg.startsWith("-doNotReplace:"))
      addADoNotReplaceFileName(arg);
    else if (arg.startsWith("-baseDirectory:"))
      parseBaseDirectory(arg);
    else
      mainDirectories.add(baseDirectory + arg);
  }

  private void addADoNotReplaceFileName(String arg) {
    String[] components = arg.split(":");
    doNotReplaceFiles.add(components[1]);
  }

  private void parseBaseDirectory(String arg) {
    baseDirectory = arg.substring(arg.indexOf(':')+1);
    if (!baseDirectory.endsWith("/"))
      baseDirectory += "/";
  }

  public List<String> getDirectories() {
    return mainDirectories;
  }

  public boolean directoriesAreValid() {
    for (String dirName : mainDirectories) {
      File checkFile = new File(dirName);
      if (!checkFile.exists()) {
        return false;
      }
    }
    return true;
  }

  public File createUpdateList() {
    for (String dirName : mainDirectories)
      addFilePathsToList(dirName);

    return FileUtil.createFile(new File("updateList"), updateListContent);

  }

  private void addFilePathsToList(String directoryPath) {
    File directory = new File(directoryPath);
    File[] files = FileUtil.getDirectoryListing(directory);
    for (File childFile : files)
      if (!isBackupFile(childFile))
        addFilePathToAppropriateList(directoryPath, childFile);

  }

  private boolean isBackupFile(File childFile) {
	return childFile.getName().endsWith(".zip");
  }

private void addFilePathToAppropriateList(String directoryPath, File childFile) {
    String childPath = directoryPath + "/" + childFile.getName();
    if (childFile.isDirectory())
      addFilePathsToList(childPath);
    else if (isDoNotReplaceFile(childFile))
      updateDoNotCopyOverContent += makePathLine(childPath);
    else
      updateListContent += makePathLine(childPath);
  }

  private String makePathLine(String path) {
    if (baseDirectory != null && baseDirectory.length() > 0 && path.startsWith(baseDirectory))
      path = path.replace(baseDirectory, "");
    if (path.startsWith("/"))
      path = path.substring(1);
    return path + "\n";
  }

  private boolean isDoNotReplaceFile(File file) {
    String name = file.getPath();
    String baseDirectoryOnOS = baseDirectory.replace( "/", File.separator);
    name = name.replace(baseDirectoryOnOS, "");
    name = name.replace(File.separator, "/");
    return doNotReplaceFiles.contains(name);
  }

  public File createDoNotUpdateList() {
    if (updateDoNotCopyOverContent.equals(""))
      for (String dirName : mainDirectories)
        addFilePathsToList(dirName);

{'fraction_non_alphanumeric': 0.11507062329920954, 'fraction_numerical': 0.0015550084229622911, 'mean_word_length': 3.7878411910669976, 'pattern_counts': {'":': 2, '<': 13, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7137304', 'n_tokens_mistral': 2264, 'n_tokens_neox': 2247, 'n_words': 491}