<commit_before>package com.pengyifan.commons.io;

import java.io.File;
import java.io.FileFilter;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;

import org.apache.commons.io.FilenameUtils;
import org.apache.commons.lang3.Validate;

/**
 * 
 * If the path is a filtered file, then process the file. If the path is a
 * directory, then process all filtered files. Otherwise throw Exception
 * 
 * @author <NAME>
 * 
 */
public abstract class BatchProcessor {

  protected final String path;
  protected final FileFilter filter;
  protected boolean debug;

  public BatchProcessor(File path, FileFilter filter) {
    this(path.toString(), filter);
  }

  public BatchProcessor(String path, FileFilter filter) {
    this.path = path;
    debug = false;
    this.filter = filter;
  }

  public final void process()
      throws Exception {

    preprocess();

    File dir = new File(path);
    List<String> basenames = null;
    if (!dir.exists()) {
      throw new IllegalArgumentException("Cannot find file/dir: " + dir);
    } else if (dir.isFile()) {
      // check if it is a filtered file
      Validate.isTrue(filter.accept(dir), "Not a %s file: %s", filter, dir);
      String basename = FilenameUtils.getBaseName(dir.getAbsolutePath());
      basenames = Collections.singletonList(basename);
      dir = new File(FilenameUtils.getFullPath(dir.getAbsolutePath()));
    } else {
      // filter
      File[] files = dir.listFiles(filter);
      Arrays.sort(files);
      basenames = new ArrayList<String>();
      for (File f : files) {
        basenames.add(FilenameUtils.getBaseName(f.getAbsolutePath()));
      }
    }
    for (String basename : basenames) {
      readResource(basename);
      preprocessFile(basename);
      processFile(basename);
      postprocessFile(basename);
    }
    postprocess();
  }

  public void setDebug(boolean debug) {
    this.debug = debug;
  }

  /**
   * Pre-process and initiates the given directory.
   */
  protected void preprocess()
      throws Exception {

  }

  /**
   * Post-processes and completes the given directory.
   */<commit_msg>fix the bug in BatchProcessor<commit_after>package com.pengyifan.commons.io;

import java.io.File;
import java.io.FileFilter;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;

import org.apache.commons.io.FilenameUtils;
import org.apache.commons.lang3.Validate;

/**
 * 
 * If the path is a filtered file, then process the file. If the path is a
 * directory, then process all filtered files. Otherwise throw Exception
 * 
 * @author <NAME>
 * 
 */
public abstract class BatchProcessor {

  protected String path;
  protected final FileFilter filter;
  protected boolean debug;

  public BatchProcessor(File path, FileFilter filter) {
    this(path.toString(), filter);
  }

  public BatchProcessor(String path, FileFilter filter) {
    this.path = path;
    debug = false;
    this.filter = filter;
  }

  public final void process()
      throws Exception {

    File dir = new File(path);
    List<String> basenames = null;
    if (!dir.exists()) {
      throw new IOException("Cannot find file/dir: " + dir);
    } else if (dir.isFile()) {
      // check if it is a filtered file
      Validate.isTrue(filter.accept(dir), "Not a %s file: %s", filter, dir);
      String basename = FilenameUtils.getBaseName(dir.getAbsolutePath());
      basenames = Collections.singletonList(basename);
      path = FilenameUtils.getFullPath(dir.getAbsolutePath());
    } else {
      // filter
      File[] files = dir.listFiles(filter);
      Arrays.sort(files);
      basenames = new ArrayList<String>();
      for (File f : files) {
        basenames.add(FilenameUtils.getBaseName(f.getAbsolutePath()));
      }
    }
    
    preprocess();
    for (String basename : basenames) {
      readResource(basename);
      preprocessFile(basename);
      processFile(basename);
      postprocessFile(basename);
    }
    postprocess();
  }

  public void setDebug(boolean debug) {
    this.debug = debug;
  }

  /**
   * Pre-process and initiates the given directory.
   */
  protected void preprocess()
      throws Exception {

  }

  /**
   * Post-processes and completes the given directory.
   */
{'fraction_non_alphanumeric': 0.11264367816091954, 'fraction_numerical': 0.0004597701149425287, 'mean_word_length': 3.108593012275732, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4937520', 'n_tokens_mistral': 1302, 'n_tokens_neox': 1280, 'n_words': 350}