<commit_before>// Copyright 2013 <NAME>. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Package htmlmin minifies HTML.
package htmlmin

import (
	"bytes"
	"io"

	"code.google.com/p/go.net/html"
	"github.com/dchest/cssmin"
	"github.com/dchest/jsmin"
)

type Options struct {
	MinifyScripts bool // if true, use jsmin to minify contents of script tags.
	MinifyStyles  bool // if true, use cssmin to minify contents of style tags and inline styles.
}

var DefaultOptions = &Options{
	MinifyScripts: false,
	MinifyStyles:  false,
}

var FullOptions = &Options{
	MinifyScripts: true,
	MinifyStyles:  true,
}

// Minify returns minified version of the given HTML data.
// If passed options is nil, uses default options.
func Minify(data []byte, options *Options) (out []byte, err error) {
	if options == nil {
		options = DefaultOptions
	}
	var b bytes.Buffer
	z := html.NewTokenizer(bytes.NewReader(data))
	raw := false
	javascript := false
	style := false
	for {
		tt := z.Next()
		switch tt {
		case html.ErrorToken:
			err := z.Err()
			if err == io.EOF {
				return b.Bytes(), nil
			}
			return nil, err
		case html.StartTagToken, html.SelfClosingTagToken:
			tagName, hasAttr := z.TagName()
			switch string(tagName) {
			case "script":
				javascript = true
				raw = true
			case "style":
				style = true
				raw = true
			case "pre", "code", "textarea":
				raw = true
			default:
				raw = false
			}
			b.WriteByte('<')
			b.Write(tagName)
			var k, v []byte
			isFirst := true
			for hasAttr {
				k, v, hasAttr = z.TagAttr()
				if javascript && string(k) == "type" && string(v) != "text/javascript" {
					javascript = false
				}
				if string(k) == "style" && options.MinifyStyles {
					v = []byte("a{" + string(v) + "}") // simulate "full" CSS
					v = cssmin.Minify(v)
					v = v[2 : len(v)-1] // strip simulation
				}
				if isFirst {
					b.WriteByte(' ')
					isFirst = false
				}
				b.Write(k)
				b.WriteByte('=')
				if quoteChar := valueQuoteChar(v); quoteChar != 0 {
					// Quoted value.
					b.WriteByte(quoteChar)
					b.WriteString(html.EscapeString(string(v)))
					b.WriteByte(quoteChar)
				} else {
					// Unquoted value.
					b.Write(v)
				}
				if hasAttr {
					b.WriteByte(' ')
				}
			}
			b.WriteByte('>')
		case html.EndTagToken:
			tagName, _ := z.TagName()
			raw = false
			if javascript && string(tagName) == "script" {
				javascript = false
			}
			if style && string(tagName) == "style" {
				style = false
			}
			b.Write([]byte("</"))
			b.Write(tagName)
			b.WriteByte('>')
		case html.CommentToken:
			if bytes.Contains(z.Raw(), []byte("<!--[if")) {
				// IE conditional comment, preserve.
				b.Write(z.Raw())
			}
			// ... otherwise, skip.
		case html.TextToken:
			if javascript && options.MinifyScripts {
				min, err := jsmin.Minify(z.Raw())
				if err != nil {
					// Just write it as is.
					b.Write(z.Raw())
				} else {
					b.Write(min)<commit_msg>Preserve comments that start with <!--//<commit_after>// Copyright 2013 <NAME>. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Package htmlmin minifies HTML.
package htmlmin

import (
	"bytes"
	"io"

	"code.google.com/p/go.net/html"
	"github.com/dchest/cssmin"
	"github.com/dchest/jsmin"
)

type Options struct {
	MinifyScripts bool // if true, use jsmin to minify contents of script tags.
	MinifyStyles  bool // if true, use cssmin to minify contents of style tags and inline styles.
}

var DefaultOptions = &Options{
	MinifyScripts: false,
	MinifyStyles:  false,
}

var FullOptions = &Options{
	MinifyScripts: true,
	MinifyStyles:  true,
}

// Minify returns minified version of the given HTML data.
// If passed options is nil, uses default options.
func Minify(data []byte, options *Options) (out []byte, err error) {
	if options == nil {
		options = DefaultOptions
	}
	var b bytes.Buffer
	z := html.NewTokenizer(bytes.NewReader(data))
	raw := false
	javascript := false
	style := false
	for {
		tt := z.Next()
		switch tt {
		case html.ErrorToken:
			err := z.Err()
			if err == io.EOF {
				return b.Bytes(), nil
			}
			return nil, err
		case html.StartTagToken, html.SelfClosingTagToken:
			tagName, hasAttr := z.TagName()
			switch string(tagName) {
			case "script":
				javascript = true
				raw = true
			case "style":
				style = true
				raw = true
			case "pre", "code", "textarea":
				raw = true
			default:
				raw = false
			}
			b.WriteByte('<')
			b.Write(tagName)
			var k, v []byte
			isFirst := true
			for hasAttr {
				k, v, hasAttr = z.TagAttr()
				if javascript && string(k) == "type" && string(v) != "text/javascript" {
					javascript = false
				}
				if string(k) == "style" && options.MinifyStyles {
					v = []byte("a{" + string(v) + "}") // simulate "full" CSS
					v = cssmin.Minify(v)
					v = v[2 : len(v)-1] // strip simulation
				}
				if isFirst {
					b.WriteByte(' ')
					isFirst = false
				}
				b.Write(k)
				b.WriteByte('=')
				if quoteChar := valueQuoteChar(v); quoteChar != 0 {
					// Quoted value.
					b.WriteByte(quoteChar)
					b.WriteString(html.EscapeString(string(v)))
					b.WriteByte(quoteChar)
				} else {
					// Unquoted value.
					b.Write(v)
				}
				if hasAttr {
					b.WriteByte(' ')
				}
			}
			b.WriteByte('>')
		case html.EndTagToken:
			tagName, _ := z.TagName()
			raw = false
			if javascript && string(tagName) == "script" {
				javascript = false
			}
			if style && string(tagName) == "style" {
				style = false
			}
			b.Write([]byte("</"))
			b.Write(tagName)
			b.WriteByte('>')
		case html.CommentToken:
			if bytes.HasPrefix(z.Raw(), []byte("<!--[if")) ||
			   bytes.HasPrefix(z.Raw(), []byte("<!--//")) {
				// Preserve IE conditional and special style comments.
				b.Write(z.Raw())
			}
			// ... otherwise, skip.
		case html.TextToken:
			if javascript && options.MinifyScripts {
				min, err := jsmin.Minify(z.Raw())
				if err != nil {
					// Just write it as is.
					b.Write(z.Raw())
				} else {
					b.Write(min)
{'fraction_non_alphanumeric': 0.14796587926509186, 'fraction_numerical': 0.0022965879265091863, 'mean_word_length': 2.620546318289786, 'pattern_counts': {'":': 6, '<': 13, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '850977', 'n_tokens_mistral': 2577, 'n_tokens_neox': 2114, 'n_words': 583}