<commit_before>/*
 * Copyright (c) 2011 <NAME> <<EMAIL>>.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *   http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 * See the NOTICE file distributed with this work for additional information
 * regarding copyright ownership.
 */
package com.tunyk.mvn.plugins.htmlcompressor;

import com.google.javascript.jscomp.CompilationLevel;
import com.google.javascript.jscomp.SourceFile;
import com.googlecode.htmlcompressor.compressor.ClosureJavaScriptCompressor;
import com.googlecode.htmlcompressor.compressor.Compressor;

import org.apache.commons.io.FileUtils;
import org.apache.maven.plugin.AbstractMojo;
import org.apache.maven.plugin.MojoExecutionException;
import org.apache.maven.plugins.annotations.LifecyclePhase;
import org.apache.maven.plugins.annotations.Mojo;
import org.apache.maven.plugins.annotations.Parameter;

import java.io.File;
import java.io.IOException;
import java.nio.charset.Charset;
import java.nio.file.Files;
import java.text.DecimalFormat;
import java.text.NumberFormat;
import java.util.ArrayList;
import java.util.List;
import java.util.regex.Pattern;
import java.util.regex.PatternSyntaxException;

/**
 * Compress HTML files
 */
@Mojo(name = "html", defaultPhase = LifecyclePhase.COMPILE, requiresProject = false)
public class HtmlCompressorMojo extends AbstractMojo {

    /**
     * file where statistics of html compression is stored
     */
    @Parameter(property="htmlcompressor.htmlCompressionStatistics", defaultValue="${project.build.directory}/htmlcompressor/html-compression-statistics.txt")
    private String htmlCompressionStatistics = "target/htmlcompressor/html-compression-statistics.txt";

    /**
     * file types to be processed
     */
    @Parameter(property="htmlcompressor.fileExt")
    private String[] fileExt;

    /**
     * if false all compression is off (default is true)
     */
    @Parameter(property="htmlcompressor.enabled", defaultValue="true")
    private Boolean enabled = true;

    /**
     * if false keeps HTML comments (default is true)
     */
    @Parameter(property="htmlcompressor.removeComments", defaultValue="true")
    private Boolean removeComments = true;

    /**
     * if false keeps multiple whitespace characters (default is true)
     */
    @Parameter(property="htmlcompressor.removeMultiSpaces", defaultValue="true")
    private Boolean removeMultiSpaces = true;

    /**
     * removes iter-tag whitespace characters
     */
    @Parameter(property="htmlcompressor.removeIntertagSpaces", defaultValue="false")
    private Boolean removeIntertagSpaces = false;

    /**
     * removes unnecessary tag attribute quotes
     */
    @Parameter(property="htmlcompressor.removeQuotes", defaultValue="false")
    private Boolean removeQuotes = false;

    /**
     * simplify existing doctype
     */
    @Parameter(property="htmlcompressor.simpleDoctype", defaultValue="false")
    private Boolean simpleDoctype = false;

    /**
     * remove optional attributes from script tags
     */
    @Parameter(property="htmlcompressor.removeScriptAttributes", defaultValue="false")
    private Boolean removeScriptAttributes = false;

    /**
     * remove optional attributes from style tags
     */
    @Parameter(property="htmlcompressor.removeStyleAttributes", defaultValue="false")
    private Boolean removeStyleAttributes = false;

    /**
     * remove optional attributes from link tags
     */
    @Parameter(property="htmlcompressor.removeLinkAttributes", defaultValue="false")
    private Boolean removeLinkAttributes = false;

    /**
     * remove optional attributes from form tags
     */
    @Parameter(property="htmlcompressor.removeFormAttributes", defaultValue="false")
    private Boolean removeFormAttributes = false;

    /**
     * remove optional attributes from input tags
     */
    @Parameter(property="htmlcompressor.removeInputAttributes", defaultValue="false")
    private Boolean removeInputAttributes = false;

    /**
     * remove values from boolean tag attributes
     */
    @Parameter(property="htmlcompressor.simpleBooleanAttributes", defaultValue="false")
    private Boolean simpleBooleanAttributes = false;

    /**
     * remove "javascript:" from inline event handlers
     */
    @Parameter(property="htmlcompressor.removeJavaScriptProtocol", defaultValue="false")
    private Boolean removeJavaScriptProtocol = false;

    /**
     * replace "http://" with "//" inside tag attributes
     */
    @Parameter(property="htmlcompressor.removeHttpProtocol", defaultValue="false")
    private Boolean removeHttpProtocol = false;

    /**
     * replace "https://" with "//" inside tag attributes
     */
    @Parameter(property="htmlcompressor.removeHttpsProtocol", defaultValue="false")
    private Boolean removeHttpsProtocol = false;

    /**
     * compress inline css
     */
    @Parameter(property="htmlcompressor.compressCss", defaultValue="false")
    private Boolean compressCss = false;

    /**
     * preserves original line breaks
     */
    @Parameter(property="htmlcompressor.preserveLineBreaks", defaultValue="false")
    private Boolean preserveLineBreaks = false;

    /**
     * --line-break param for Yahoo YUI Compressor
     */
    @Parameter(property="htmlcompressor.yuiCssLineBreak", defaultValue="-1")
    private Integer yuiCssLineBreak = -1;

    /**
     * css compressor
     */
    // TODO Unsupported
    @Parameter(property="htmlcompressor.cssCompressor", defaultValue="")
    private Compressor cssCompressor;

    /**
     * compress inline javascript
     */
    @Parameter(property="htmlcompressor.compressJavaScript", defaultValue="false")
    private Boolean compressJavaScript = false;

    /**
     * javascript compression: "yui" or "closure"
     */
    @Parameter(property="htmlcompressor.jsCompressor", defaultValue="yui")
    private String jsCompressor = "yui";

    /**
     * javascript compression
     */
    // TODO Unsupported
    @Parameter(property="htmlcompressor.javaScriptCompressor", defaultValue="")
    private Compressor javaScriptCompressor;

    /**
     * --nomunge param for Yahoo YUI Compressor
     */
    @Parameter(property="htmlcompressor.yuiJsNoMunge", defaultValue="false")
    private Boolean yuiJsNoMunge = false;

    /**
     * --preserve-semi param for Yahoo YUI Compressor
     */
    @Parameter(property="htmlcompressor.yuiJsPreserveAllSemiColons", defaultValue="false")
    private Boolean yuiJsPreserveAllSemiColons = false;

    /**
     * --line-break param for Yahoo YUI Compressor
     */
    @Parameter(property="htmlcompressor.yuiJsLineBreak", defaultValue="-1")
    private Integer yuiJsLineBreak = -1;

    /**
     * closureOptLevel = "simple", "advanced" or "whitespace"
     */
    @Parameter(property="htmlcompressor.closureOptLevel", defaultValue="simple")
    private String closureOptLevel = "simple";

    /**
     * --disable-optimizations param for Yahoo YUI Compressor
     */
    @Parameter(property="htmlcompressor.yuiJsDisableOptimizations", defaultValue="false")
    private Boolean yuiJsDisableOptimizations = false;

    /**
     * predefined patterns for most often used custom preservation rules: PHP_TAG_PATTERN and SERVER_SCRIPT_TAG_PATTERN.
     */
    @Parameter(property="htmlcompressor.predefinedPreservePatterns")
    private String[] predefinedPreservePatterns;

    /**
     * preserve patterns
     */
    @Parameter(property="htmlcompressor.preservePatterns")
    private String[] preservePatterns;

    /**
     * list of files containing preserve patterns
     */
    @Parameter(property="htmlcompressor.preservePatternFiles")
    private File[] preservePatternFiles;

    /**
     * HTML compression statistics
     */
    @Parameter(property="htmlcompressor.generateStatistics", defaultValue="true")
    private Boolean generateStatistics = true;

    /**
     * source folder where html files are located.
     */
    @Parameter(property="htmlcompressor.srcFolder", defaultValue="${basedir}/src/main/resources")
    private String srcFolder = "src/main/resources";

    /**
     * target folder where compressed html files will be placed.
     */
    @Parameter(property="htmlcompressor.targetFolder", defaultValue="${project.build.directory}/classes")
    private String targetFolder = "target/classes";

    /**
     * Create javascript file which includes all compressed html files as json object. If set to true then javascriptHtmlSpriteIntegrationFile param is required, otherwise it will throw exception.
     */
    @Parameter(property="htmlcompressor.javascriptHtmlSprite", defaultValue="true")
    private Boolean javascriptHtmlSprite = true;

    /**
     * JavaScript sprite integration file (first occurrence of "%s" will be substituted by json with all compressed html strings)
     */
    @Parameter(property="htmlcompressor.javascriptHtmlSpriteIntegrationFile", defaultValue="${basedir}/src/main/resources/html/integration.js")
    private String javascriptHtmlSpriteIntegrationFile = "src/main/resources/html/integration.js";

    /**
     * The target JavaScript sprite file with compressed html files as json object.
     */
    @Parameter(property="htmlcompressor.javascriptHtmlSpriteTargetFile", defaultValue="${project.build.directory}/htmlcompressor/html/integration.js")
    private String javascriptHtmlSpriteTargetFile = "target/htmlcompressor/html/integration.js";

    /**
     * Charset encoding for files to read and create
     */
    @Parameter(property="htmlcompressor.encoding", defaultValue="utf-8")
    private String encoding = "utf-8";

    /**
     * Disable default built-in closure externs.
     */
    @Parameter(property="htmlcompressor.closureCustomExternsOnly", defaultValue="false")
    private Boolean closureCustomExternsOnly = false;

    /**
     * Sets custom closure externs file list.
     */
    @Parameter(property="htmlcompressor.closureExterns")
    private String[] closureExterns;

    @Override
    public void execute() throws MojoExecutionException {
        if (!enabled) {
            getLog().info("HTML compression was turned off.");
            return;
        }

        getLog().info("Compressing " + srcFolder);
        HtmlCompressor htmlCompressor = new HtmlCompressor(srcFolder, targetFolder);
<commit_msg>[enhance] If directory passed in doesn't exist exit gracefuly<commit_after>/*
 * Copyright (c) 2011 <NAME> <<EMAIL>>.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *   http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 * See the NOTICE file distributed with this work for additional information
 * regarding copyright ownership.
 */
package com.tunyk.mvn.plugins.htmlcompressor;

import com.google.javascript.jscomp.CompilationLevel;
import com.google.javascript.jscomp.SourceFile;
import com.googlecode.htmlcompressor.compressor.ClosureJavaScriptCompressor;
import com.googlecode.htmlcompressor.compressor.Compressor;

import org.apache.commons.io.FileUtils;
import org.apache.maven.plugin.AbstractMojo;
import org.apache.maven.plugin.MojoExecutionException;
import org.apache.maven.plugins.annotations.LifecyclePhase;
import org.apache.maven.plugins.annotations.Mojo;
import org.apache.maven.plugins.annotations.Parameter;

import java.io.File;
import java.io.IOException;
import java.nio.charset.Charset;
import java.nio.file.Files;
import java.text.DecimalFormat;
import java.text.NumberFormat;
import java.util.ArrayList;
import java.util.List;
import java.util.regex.Pattern;
import java.util.regex.PatternSyntaxException;

/**
 * Compress HTML files
 */
@Mojo(name = "html", defaultPhase = LifecyclePhase.COMPILE, requiresProject = false)
public class HtmlCompressorMojo extends AbstractMojo {

    /**
     * file where statistics of html compression is stored
     */
    @Parameter(property="htmlcompressor.htmlCompressionStatistics", defaultValue="${project.build.directory}/htmlcompressor/html-compression-statistics.txt")
    private String htmlCompressionStatistics = "target/htmlcompressor/html-compression-statistics.txt";

    /**
     * file types to be processed
     */
    @Parameter(property="htmlcompressor.fileExt")
    private String[] fileExt;

    /**
     * if false all compression is off (default is true)
     */
    @Parameter(property="htmlcompressor.enabled", defaultValue="true")
    private Boolean enabled = true;

    /**
     * if false keeps HTML comments (default is true)
     */
    @Parameter(property="htmlcompressor.removeComments", defaultValue="true")
    private Boolean removeComments = true;

    /**
     * if false keeps multiple whitespace characters (default is true)
     */
    @Parameter(property="htmlcompressor.removeMultiSpaces", defaultValue="true")
    private Boolean removeMultiSpaces = true;

    /**
     * removes iter-tag whitespace characters
     */
    @Parameter(property="htmlcompressor.removeIntertagSpaces", defaultValue="false")
    private Boolean removeIntertagSpaces = false;

    /**
     * removes unnecessary tag attribute quotes
     */
    @Parameter(property="htmlcompressor.removeQuotes", defaultValue="false")
    private Boolean removeQuotes = false;

    /**
     * simplify existing doctype
     */
    @Parameter(property="htmlcompressor.simpleDoctype", defaultValue="false")
    private Boolean simpleDoctype = false;

    /**
     * remove optional attributes from script tags
     */
    @Parameter(property="htmlcompressor.removeScriptAttributes", defaultValue="false")
    private Boolean removeScriptAttributes = false;

    /**
     * remove optional attributes from style tags
     */
    @Parameter(property="htmlcompressor.removeStyleAttributes", defaultValue="false")
    private Boolean removeStyleAttributes = false;

    /**
     * remove optional attributes from link tags
     */
    @Parameter(property="htmlcompressor.removeLinkAttributes", defaultValue="false")
    private Boolean removeLinkAttributes = false;

    /**
     * remove optional attributes from form tags
     */
    @Parameter(property="htmlcompressor.removeFormAttributes", defaultValue="false")
    private Boolean removeFormAttributes = false;

    /**
     * remove optional attributes from input tags
     */
    @Parameter(property="htmlcompressor.removeInputAttributes", defaultValue="false")
    private Boolean removeInputAttributes = false;

    /**
     * remove values from boolean tag attributes
     */
    @Parameter(property="htmlcompressor.simpleBooleanAttributes", defaultValue="false")
    private Boolean simpleBooleanAttributes = false;

    /**
     * remove "javascript:" from inline event handlers
     */
    @Parameter(property="htmlcompressor.removeJavaScriptProtocol", defaultValue="false")
    private Boolean removeJavaScriptProtocol = false;

    /**
     * replace "http://" with "//" inside tag attributes
     */
    @Parameter(property="htmlcompressor.removeHttpProtocol", defaultValue="false")
    private Boolean removeHttpProtocol = false;

    /**
     * replace "https://" with "//" inside tag attributes
     */
    @Parameter(property="htmlcompressor.removeHttpsProtocol", defaultValue="false")
    private Boolean removeHttpsProtocol = false;

    /**
     * compress inline css
     */
    @Parameter(property="htmlcompressor.compressCss", defaultValue="false")
    private Boolean compressCss = false;

    /**
     * preserves original line breaks
     */
    @Parameter(property="htmlcompressor.preserveLineBreaks", defaultValue="false")
    private Boolean preserveLineBreaks = false;

    /**
     * --line-break param for Yahoo YUI Compressor
     */
    @Parameter(property="htmlcompressor.yuiCssLineBreak", defaultValue="-1")
    private Integer yuiCssLineBreak = -1;

    /**
     * css compressor
     */
    // TODO Unsupported
    @Parameter(property="htmlcompressor.cssCompressor", defaultValue="")
    private Compressor cssCompressor;

    /**
     * compress inline javascript
     */
    @Parameter(property="htmlcompressor.compressJavaScript", defaultValue="false")
    private Boolean compressJavaScript = false;

    /**
     * javascript compression: "yui" or "closure"
     */
    @Parameter(property="htmlcompressor.jsCompressor", defaultValue="yui")
    private String jsCompressor = "yui";

    /**
     * javascript compression
     */
    // TODO Unsupported
    @Parameter(property="htmlcompressor.javaScriptCompressor", defaultValue="")
    private Compressor javaScriptCompressor;

    /**
     * --nomunge param for Yahoo YUI Compressor
     */
    @Parameter(property="htmlcompressor.yuiJsNoMunge", defaultValue="false")
    private Boolean yuiJsNoMunge = false;

    /**
     * --preserve-semi param for Yahoo YUI Compressor
     */
    @Parameter(property="htmlcompressor.yuiJsPreserveAllSemiColons", defaultValue="false")
    private Boolean yuiJsPreserveAllSemiColons = false;

    /**
     * --line-break param for Yahoo YUI Compressor
     */
    @Parameter(property="htmlcompressor.yuiJsLineBreak", defaultValue="-1")
    private Integer yuiJsLineBreak = -1;

    /**
     * closureOptLevel = "simple", "advanced" or "whitespace"
     */
    @Parameter(property="htmlcompressor.closureOptLevel", defaultValue="simple")
    private String closureOptLevel = "simple";

    /**
     * --disable-optimizations param for Yahoo YUI Compressor
     */
    @Parameter(property="htmlcompressor.yuiJsDisableOptimizations", defaultValue="false")
    private Boolean yuiJsDisableOptimizations = false;

    /**
     * predefined patterns for most often used custom preservation rules: PHP_TAG_PATTERN and SERVER_SCRIPT_TAG_PATTERN.
     */
    @Parameter(property="htmlcompressor.predefinedPreservePatterns")
    private String[] predefinedPreservePatterns;

    /**
     * preserve patterns
     */
    @Parameter(property="htmlcompressor.preservePatterns")
    private String[] preservePatterns;

    /**
     * list of files containing preserve patterns
     */
    @Parameter(property="htmlcompressor.preservePatternFiles")
    private File[] preservePatternFiles;

    /**
     * HTML compression statistics
     */
    @Parameter(property="htmlcompressor.generateStatistics", defaultValue="true")
    private Boolean generateStatistics = true;

    /**
     * source folder where html files are located.
     */
    @Parameter(property="htmlcompressor.srcFolder", defaultValue="${basedir}/src/main/resources")
    private String srcFolder = "src/main/resources";

    /**
     * target folder where compressed html files will be placed.
     */
    @Parameter(property="htmlcompressor.targetFolder", defaultValue="${project.build.directory}/classes")
    private String targetFolder = "target/classes";

    /**
     * Create javascript file which includes all compressed html files as json object. If set to true then javascriptHtmlSpriteIntegrationFile param is required, otherwise it will throw exception.
     */
    @Parameter(property="htmlcompressor.javascriptHtmlSprite", defaultValue="true")
    private Boolean javascriptHtmlSprite = true;

    /**
     * JavaScript sprite integration file (first occurrence of "%s" will be substituted by json with all compressed html strings)
     */
    @Parameter(property="htmlcompressor.javascriptHtmlSpriteIntegrationFile", defaultValue="${basedir}/src/main/resources/html/integration.js")
    private String javascriptHtmlSpriteIntegrationFile = "src/main/resources/html/integration.js";

    /**
     * The target JavaScript sprite file with compressed html files as json object.
     */
    @Parameter(property="htmlcompressor.javascriptHtmlSpriteTargetFile", defaultValue="${project.build.directory}/htmlcompressor/html/integration.js")
    private String javascriptHtmlSpriteTargetFile = "target/htmlcompressor/html/integration.js";

    /**
     * Charset encoding for files to read and create
     */
    @Parameter(property="htmlcompressor.encoding", defaultValue="utf-8")
    private String encoding = "utf-8";

    /**
     * Disable default built-in closure externs.
     */
    @Parameter(property="htmlcompressor.closureCustomExternsOnly", defaultValue="false")
    private Boolean closureCustomExternsOnly = false;

    /**
     * Sets custom closure externs file list.
     */
    @Parameter(property="htmlcompressor.closureExterns")
    private String[] closureExterns;

    @Override
    public void execute() throws MojoExecutionException {
        if (!enabled) {
            getLog().info("HTML compression was turned off.");
            return;
        }

        if (!new File(srcFolder).exists()) {
            getLog().warn("Compressor folder does not exist, skipping compression of " + srcFolder);
            return;
        }

        getLog().info("Compressing " + srcFolder);
        HtmlCompressor htmlCompressor = new HtmlCompressor(srcFolder, targetFolder);

{'fraction_non_alphanumeric': 0.10404464326892035, 'fraction_numerical': 0.0012913342249688696, 'mean_word_length': 3.8090485695276115, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 2, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1567633', 'n_tokens_mistral': 5737, 'n_tokens_neox': 5475, 'n_words': 1393}