<commit_before>package util;

/**
 * Utilities class
 * The purpose of this class is to take all html files containing in the targeted directory
 * and to delete the line staring by ~~~
 * The resulting file is outputed as 
 *     _<originalFilename.html>
 * Typically it is used to removed unwanted content from html file. In the original file we will
 * have 
 * <!-- 
 * ~~~ -->
 * <section to remove by this tool>
 * <!-- -->
 */
public class FilterDocFiles {

  public static void main(String[] args) throws java.io.IOException {
    if (args.length == 0) {
      System.out.println("Missing target directory");
      System.exit(-1);
    }
    java.io.File sourceDir = new java.io.File(args[0]);
    filter(sourceDir);
  }
  
  
  private static void filterFile(java.io.File file) throws java.io.IOException {
    String name = file.getName();
    if (! name.endsWith(".html")) return;
    if (name.startsWith("_")) return;
    System.out.println("Processing "+file);
    byte[] b = getBytesFromInputStream(new java.io.FileInputStream(file));
    String html = new String(b);
    html = removeMarkedLine(html);
    html = changeImagesPath(html);
    b = html.getBytes();
    java.io.File newFile = new java.io.File(file.getParentFile(), "_"+file.getName());
    newFile.delete();
    java.io.OutputStream out = new java.io.BufferedOutputStream(new java.io.FileOutputStream(newFile));
    out.write(b, 0, b.length);
    out.flush();
    out.close();<commit_msg>feature added<commit_after>package util;

/**
 * Utilities class
 * The purpose of this class is to take all html files containing in the targeted directory
 * and to delete the line staring by ~~~
 * The resulting file is outputed as 
 *     _<originalFilename.html>
 * Typically it is used to removed unwanted content from html file. In the original file we will
 * have 
 * <!-- 
 * ~~~ -->
 * <section to remove by this tool>
 * <!-- -->
 */
public class FilterDocFiles {
	
	static boolean changeImagePath = true;

  public static void main(String[] args) throws java.io.IOException {
    if (args.length == 0) {
      System.out.println("Missing target directory");
      System.exit(-1);
    }
    if (System.getProperty("changeImages").equals("false")) changeImagePath=false;
    java.io.File sourceDir = new java.io.File(args[0]);
    filter(sourceDir);
  }
  
  
  private static void filterFile(java.io.File file) throws java.io.IOException {
    String name = file.getName();
    if (! name.endsWith(".html")) return;
    if (name.startsWith("_")) return;
    System.out.println("Processing "+file);
    byte[] b = getBytesFromInputStream(new java.io.FileInputStream(file));
    String html = new String(b);
    html = removeMarkedLine(html);
    html = changeImagesPath(html);
    b = html.getBytes();
    java.io.File newFile = new java.io.File(file.getParentFile(), "_"+file.getName());
    newFile.delete();
    java.io.OutputStream out = new java.io.BufferedOutputStream(new java.io.FileOutputStream(newFile));
    out.write(b, 0, b.length);
    out.flush();
    out.close();
{'fraction_non_alphanumeric': 0.13673536146548904, 'fraction_numerical': 0.0026169447170428526, 'mean_word_length': 3.4126984126984126, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '216189', 'n_tokens_mistral': 909, 'n_tokens_neox': 893, 'n_words': 263}