<commit_before>package io.github.codemumbler.maven.plugin;

import com.google.javascript.jscomp.*;
import org.apache.maven.plugin.AbstractMojo;
import org.apache.maven.plugin.MojoExecutionException;
import org.apache.maven.plugins.annotations.LifecyclePhase;
import org.apache.maven.plugins.annotations.Mojo;
import org.apache.maven.plugins.annotations.Parameter;
import org.codehaus.plexus.util.IOUtil;

import java.io.File;
import java.io.FileFilter;
import java.io.FileInputStream;
import java.io.FileWriter;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.logging.Level;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

@Mojo(name = "minify-js", defaultPhase = LifecyclePhase.PREPARE_PACKAGE)
public class MinifyJSMojo
    extends AbstractMojo {

  private static final String PATH_SEPARATOR = System.getProperty("file.separator");
  @Parameter(defaultValue = "${project.basedir}/src/main/webapp/js/lib")
  private File externalJsDirectory;

  @Parameter(defaultValue = "${project.basedir}/src/main/webapp/js")
  private File jsDirectory;

  @Parameter(defaultValue = "${project.build.directory}/${project.build.finalName}/js/combined.min.js")
  private String outputFilename;

  @Parameter(defaultValue = "false")
  private boolean compile;

  @Parameter(defaultValue = "${project.basedir}/src/main/webapp")
  private File htmlSourceDirectory;

  @Parameter(defaultValue = "${project.build.directory}/${project.build.finalName}")
  private String htmlOutputDirectory;

  @Parameter(defaultValue = "true")
  private boolean updateHTML;

  public void execute() throws MojoExecutionException {
    com.google.javascript.jscomp.Compiler.setLoggingLevel(Level.INFO);
    com.google.javascript.jscomp.Compiler compiler = new com.google.javascript.jscomp.Compiler();

    CompilerOptions options = new CompilerOptions();
    if (!compile) {
      CompilationLevel.WHITESPACE_ONLY.setOptionsForCompilationLevel(options);
    } else {
      CompilationLevel.SIMPLE_OPTIMIZATIONS.setOptionsForCompilationLevel(options);
    }
    WarningLevel.VERBOSE.setOptionsForWarningLevel(options);

    List<SourceFile> externalJavascriptFiles = new ArrayList<>();
    if ( externalJsDirectory != null && externalJsDirectory.listFiles() != null ) {
      File[] externalJSFiles = externalJsDirectory.listFiles(new FileFilter() {
        @Override public boolean accept(File file) {
          return !file.isDirectory() && file.getName().endsWith(".js");
        }
      });
      for (File file : externalJSFiles) {
        externalJavascriptFiles.add(JSSourceFile.fromFile(file.getAbsolutePath()));
      }
    }

    List<SourceFile> projectSourceFiles = new ArrayList<>();
    for (File file : getAllFilesMatchingPattern(jsDirectory)) {
      projectSourceFiles.add(JSSourceFile.fromFile(file.getAbsolutePath()));
    }

    compiler.compile(externalJavascriptFiles, projectSourceFiles, options);

    for (JSError message : compiler.getWarnings()) {
      System.err.println("Warning message: " + message.toString());
    }

    for (JSError message : compiler.getErrors()) {
      System.err.println("Error message: " + message.toString());
    }
    String outputPath = outputFilename.substring(0, outputFilename.lastIndexOf("/"));
    new File(outputPath).mkdirs();
    try (FileWriter outputFile = new FileWriter(outputFilename)) {
      outputFile.write(compiler.toSource());
    } catch (Exception e) {
      throw new MojoExecutionException("Error while writing minified file", e);
    }
    if (updateHTML) {
      try {
        updateHTMLJSReferences(projectSourceFiles);
      } catch (Exception e) {
        throw new MojoExecutionException("Error updating HTML files", e);
      }
    }
  }

  private List<File> getAllFilesMatchingPattern(final File parentDirectory) {
    List<File> files = new ArrayList<>();
    for (File file : parentDirectory.listFiles()) {
      if (file.isDirectory() && !file.getAbsolutePath().equals(externalJsDirectory.getAbsolutePath())) {
        files.addAll(getAllFilesMatchingPattern(file));
      }
      if (file.getName().endsWith(".js")) {
        files.add(file);
      }
    }
    return files;
  }

  private void updateHTMLJSReferences(List<SourceFile> projectSourceFiles) throws Exception {
    File[] htmlFiles = htmlSourceDirectory.listFiles(new FileFilter() {
      @Override
      public boolean accept(File pathname) {
        return (pathname.getName().endsWith(".html"));
      }
    });

    for (File htmlFile : htmlFiles) {
      String content = loadFileAsString(htmlFile);
      boolean replaced = false;
      for (SourceFile jsSourceFile : projectSourceFiles) {
        String src = jsSourceFile.getName().replace(htmlFile.getParentFile().getAbsolutePath() + PATH_SEPARATOR, "");
        src = src.replace("\\", "/");
        Pattern pattern = Pattern.compile("<script.*?src=\"" + src + "\".*?></script>");
        Matcher matcher = pattern.matcher(content);
        if (replaced) {
          content = content.replaceAll("\\s*<script.*?src=\"" + src + "\".*?></script>", "");
          continue;
        }
        if (matcher.find()) {<commit_msg>code clean up<commit_after>package io.github.codemumbler.maven.plugin;

import com.google.javascript.jscomp.*;
import org.apache.maven.plugin.AbstractMojo;
import org.apache.maven.plugin.MojoExecutionException;
import org.apache.maven.plugins.annotations.LifecyclePhase;
import org.apache.maven.plugins.annotations.Mojo;
import org.apache.maven.plugins.annotations.Parameter;
import org.codehaus.plexus.util.IOUtil;

import java.io.File;
import java.io.FileFilter;
import java.io.FileInputStream;
import java.io.FileWriter;
import java.util.ArrayList;
import java.util.List;
import java.util.logging.Level;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

@Mojo(name = "minify-js", defaultPhase = LifecyclePhase.PREPARE_PACKAGE)
public class MinifyJSMojo
    extends AbstractMojo {

  private static final String PATH_SEPARATOR = System.getProperty("file.separator");
  @SuppressWarnings("unused")
  @Parameter(defaultValue = "${project.basedir}/src/main/webapp/js/lib")
  private File externalJsDirectory;

  @SuppressWarnings("unused")
  @Parameter(defaultValue = "${project.basedir}/src/main/webapp/js")
  private File jsDirectory;

  @SuppressWarnings("unused")
  @Parameter(defaultValue = "${project.build.directory}/${project.build.finalName}/js/combined.min.js")
  private String outputFilename;

  @SuppressWarnings("unused")
  @Parameter(defaultValue = "false")
  private boolean compile;

  @SuppressWarnings("unused")
  @Parameter(defaultValue = "${project.basedir}/src/main/webapp")
  private File htmlSourceDirectory;

  @SuppressWarnings("unused")
  @Parameter(defaultValue = "${project.build.directory}/${project.build.finalName}")
  private String htmlOutputDirectory;

  @SuppressWarnings("unused")
  @Parameter(defaultValue = "true")
  private boolean updateHTML;

  public void execute() throws MojoExecutionException {
    com.google.javascript.jscomp.Compiler.setLoggingLevel(Level.INFO);
    com.google.javascript.jscomp.Compiler compiler = new com.google.javascript.jscomp.Compiler();

    CompilerOptions options = new CompilerOptions();
    if (!compile) {
      CompilationLevel.WHITESPACE_ONLY.setOptionsForCompilationLevel(options);
    } else {
      CompilationLevel.SIMPLE_OPTIMIZATIONS.setOptionsForCompilationLevel(options);
    }
    WarningLevel.VERBOSE.setOptionsForWarningLevel(options);

    List<SourceFile> externalJavascriptFiles = new ArrayList<>();
    if ( externalJsDirectory != null && externalJsDirectory.listFiles() != null ) {
      File[] externalJSFiles = externalJsDirectory.listFiles(new FileFilter() {
        @Override public boolean accept(File file) {
          return !file.isDirectory() && file.getName().endsWith(".js");
        }
      });
      for (File file : externalJSFiles) {
        externalJavascriptFiles.add(JSSourceFile.fromFile(file.getAbsolutePath()));
      }
    }

    List<SourceFile> projectSourceFiles = new ArrayList<>();
    for (File file : listFilesInSubDirectories(jsDirectory)) {
      projectSourceFiles.add(JSSourceFile.fromFile(file.getAbsolutePath()));
    }

    compiler.compile(externalJavascriptFiles, projectSourceFiles, options);

    for (JSError message : compiler.getWarnings()) {
      System.err.println("Warning message: " + message.toString());
    }

    for (JSError message : compiler.getErrors()) {
      System.err.println("Error message: " + message.toString());
    }
    String outputPath = outputFilename.substring(0, outputFilename.lastIndexOf("/"));
    new File(outputPath).mkdirs();
    try (FileWriter outputFile = new FileWriter(outputFilename)) {
      outputFile.write(compiler.toSource());
    } catch (Exception e) {
      throw new MojoExecutionException("Error while writing minified file", e);
    }
    if (updateHTML) {
      try {
        updateHTMLJSReferences(projectSourceFiles);
      } catch (Exception e) {
        throw new MojoExecutionException("Error updating HTML files", e);
      }
    }
  }

  private List<File> listFilesInSubDirectories(final File parentDirectory) {
    List<File> files = new ArrayList<>();
    File[] listOfFiles = parentDirectory.listFiles();
    if (listOfFiles != null) {
      for (File file : listOfFiles) {
        if (file.isDirectory() && !file.getAbsolutePath().equals(externalJsDirectory.getAbsolutePath())) {
          files.addAll(listFilesInSubDirectories(file));
        }
        if (file.getName().endsWith(".js")) {
          files.add(file);
        }
      }
    }
    return files;
  }

  private void updateHTMLJSReferences(List<SourceFile> projectSourceFiles) throws Exception {
    File[] htmlFiles = htmlSourceDirectory.listFiles(new FileFilter() {
      @Override
      public boolean accept(File pathname) {
        return (pathname.getName().endsWith(".html"));
      }
    });

    for (File htmlFile : htmlFiles) {
      String content = loadFileAsString(htmlFile);
      boolean replaced = false;
      for (SourceFile jsSourceFile : projectSourceFiles) {
        String src = jsSourceFile.getName().replace(htmlFile.getParentFile().getAbsolutePath() + PATH_SEPARATOR, "");
        src = src.replace("\\", "/");
        Pattern pattern = Pattern.compile("<script.*?src=\"" + src + "\".*?></script>");
        Matcher matcher = pattern.matcher(content);
        if (replaced) {
          content = content.replaceAll("\\s*<script.*?src=\"" + src + "\".*?></script>", "");
          continue;
        }
        if (matcher.find()) {
{'fraction_non_alphanumeric': 0.12993870815652994, 'fraction_numerical': 0.0001885902876001886, 'mean_word_length': 3.993408662900188, 'pattern_counts': {'":': 0, '<': 27, '<?xml version=': 0, '>': 27, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1565176', 'n_tokens_mistral': 3035, 'n_tokens_neox': 2988, 'n_words': 555}