<commit_before>package air

import (
	"errors"
	"html/template"
	"io"
	"io/ioutil"
	"path/filepath"
	"reflect"
	"regexp"
	"strings"
	"time"
)

// renderer is used to provide a `render()` method for an `Air` instance
// for renders a "text/html" response by using `template.Template`.
type renderer struct {
	goTemplate      *template.Template
	templateFuncMap template.FuncMap
	air             *Air
}

// defaultTemplateFuncMap is a default template func map of `renderer`
var defaultTemplateFuncMap template.FuncMap

func init() {
	tfm := make(template.FuncMap)

	tfm["strlen"] = strlen
	tfm["substr"] = substr
	tfm["str2html"] = str2html
	tfm["html2str"] = html2str
	tfm["datefmt"] = datefmt
	tfm["eq"] = eq
	tfm["ne"] = ne
	tfm["lt"] = lt
	tfm["le"] = le
	tfm["gt"] = gt
	tfm["ge"] = ge

	defaultTemplateFuncMap = tfm
}

// newRenderer returns a new instance of `Renderer`.
func newRenderer(a *Air) *renderer {
	return &renderer{
		templateFuncMap: defaultTemplateFuncMap,
		air:             a,
	}
}

// parseTemplates parses files into templates.
//
// e.g. r.air.Config.TemplatesRoot == "templates"
//
// templates/
//   index.html
//   login.html
//   register.html
//
// templates/parts/
//   header.html
//   footer.html
//
// will be parsed into:
//
// "index.html", "login.html", "register.html",
// "parts/header.html", "parts/footer.html".
func (r *renderer) parseTemplates() {
	tr := filepath.Clean(r.air.Config.TemplatesRoot)
	filenames, err := filepath.Glob(tr + "/*/*.html")
	if err != nil {
		panic(err)
	}
	for _, filename := range filenames {
		b, err := ioutil.ReadFile(filename)
		if err != nil {
			panic(err)
		}
		name := filepath.ToSlash(filename[len(tr):])
		if name[0] == '/' {
			name = name[1:]
		}
		var tmpl *template.Template
		if r.goTemplate == nil {
			r.goTemplate = template.New(name).Funcs(r.templateFuncMap)
		}
		if name == r.goTemplate.Name() {
			tmpl = r.goTemplate
		} else {
			tmpl = r.goTemplate.New(name)
		}
		_, err = tmpl.Parse(string(b))
		if err != nil {
			panic(err)
		}
	}
}

// render renders a "text/html" response by using `template.Template`
func (r *renderer) render(wr io.Writer, templateName string, c *Context) error {
	return r.goTemplate.ExecuteTemplate(wr, templateName, c.Data)
}

// Basic type kind
type typeKind int

const (
	invalidKind typeKind = iota
	intKind
	uintKind
	floatKind
	complexKind
	boolKind
	stringKind
)

// Template func error
var (
	errBadComparisonType = errors.New("invalid type for comparison")
	errBadComparison     = errors.New("incompatible types for comparison")
	errNoComparison      = errors.New("missing argument for comparison")
)

// strlen returns the number of characters in s.
func strlen(s string) int {
	return len([]rune(s))
}

// substr returns the substring from start to length.
func substr(s string, start, length int) string {
	bt := []rune(s)
	if start < 0 {
		start = 0
	}
	if start > len(bt) {
		start = start % len(bt)
	}
	var end int
	if (start + length) > (len(bt) - 1) {
		end = len(bt)
	} else {
		end = start + length
	}
	return string(bt[start:end])
}

// html2str returns escaping text convert from html.
func html2str(html string) string {
	src := string(html)

	re, _ := regexp.Compile("\\<[\\S\\s]+?\\>")
	src = re.ReplaceAllStringFunc(src, strings.ToLower)

	//remove STYLE
	re, _ = regexp.Compile("\\<style[\\S\\s]+?\\</style\\>")
	src = re.ReplaceAllString(src, "")

	//remove SCRIPT
	re, _ = regexp.Compile("\\<script[\\S\\s]+?\\</script\\>")
	src = re.ReplaceAllString(src, "")

	re, _ = regexp.Compile("\\<[\\S\\s]+?\\>")
	src = re.ReplaceAllString(src, "\n")

	re, _ = regexp.Compile("\\s{2,}")
	src = re.ReplaceAllString(src, "\n")

	return strings.TrimSpace(src)
}

// str2html returns the `template.HTML` convert from raw.
func str2html(raw string) template.HTML {
	return template.HTML(raw)
}

// datefmt takes a time and a layout string and returns a string with the formatted date.
func datefmt(t time.Time, layout string) string {
	return t.Format(layout)
}

// basicKind returns a basic type kind with a provided `reflect.Value`.
func basicKind(v reflect.Value) (typeKind, error) {
	switch v.Kind() {
	case reflect.Bool:
		return boolKind, nil
	case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
		return intKind, nil
	case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr:
		return uintKind, nil
	case reflect.Float32, reflect.Float64:
		return floatKind, nil
	case reflect.Complex64, reflect.Complex128:
		return complexKind, nil
	case reflect.String:
		return stringKind, nil
	}
	return invalidKind, errBadComparisonType
}

// eq evaluates the comparison a == b || a == c || ...
func eq(arg1 interface{}, arg2 ...interface{}) (bool, error) {
	v1 := reflect.ValueOf(arg1)
	k1, err := basicKind(v1)
	if err != nil {
		return false, err
	}
	if len(arg2) == 0 {
		return false, errNoComparison
	}
	for _, arg := range arg2 {
		v2 := reflect.ValueOf(arg)
		k2, err := basicKind(v2)
		if err != nil {
			return false, err
		}
		if k1 != k2 {
			return false, errBadComparison
		}
		truth := false
		switch k1 {
		case boolKind:
			truth = v1.Bool() == v2.Bool()
		case complexKind:
			truth = v1.Complex() == v2.Complex()
		case floatKind:
			truth = v1.Float() == v2.Float()
		case intKind:
			truth = v1.Int() == v2.Int()
		case stringKind:
			truth = v1.String() == v2.String()
		case uintKind:
			truth = v1.Uint() == v2.Uint()
		default:
			panic("invalid kind")
		}
		if truth {
			return true, nil
		}
	}
	return false, nil
}

// ne evaluates the comparison a != b && a != c && ...
func ne(arg1 interface{}, arg2 ...interface{}) (bool, error) {
	// != is the inverse of ==.
	equal, err := eq(arg1, arg2)
	return !equal, err
}

// lt evaluates the comparison a < b.
func lt(arg1, arg2 interface{}) (bool, error) {
	v1 := reflect.ValueOf(arg1)
	k1, err := basicKind(v1)
	if err != nil {
		return false, err
	}
	v2 := reflect.ValueOf(arg2)
	k2, err := basicKind(v2)
	if err != nil {
		return false, err
	}
	if k1 != k2 {
		return false, errBadComparison
	}
	truth := false
	switch k1 {
	case boolKind, complexKind:
		return false, errBadComparisonType
	case floatKind:
		truth = v1.Float() < v2.Float()
	case intKind:
		truth = v1.Int() < v2.Int()
	case stringKind:
		truth = v1.String() < v2.String()
	case uintKind:
		truth = v1.Uint() < v2.Uint()
	default:
		panic("invalid kind")
	}
	return truth, nil
}

// le evaluates the comparison <= b.
func le(arg1, arg2 interface{}) (bool, error) {
	// <= is < or ==.
	lessThan, err := lt(arg1, arg2)
	if lessThan || err != nil {
		return lessThan, err
	}
	return eq(arg1, arg2)
}

// gt evaluates the comparison a > b.
func gt(arg1, arg2 interface{}) (bool, error) {
	// > is the inverse of <=.
	lessOrEqual, err := le(arg1, arg2)
	if err != nil {
		return false, err
	}
	return !lessOrEqual, nil
}

// ge evaluates the comparison a >= b.
func ge(arg1, arg2 interface{}) (bool, error) {
	// >= is the inverse of <.
	lessThan, err := lt(arg1, arg2)
	if err != nil {
		return false, err
	}
	return !lessThan, nil
}<commit_msg>refactor: redesign all template funcs<commit_after>package air

import (
	"html/template"
	"io"
	"io/ioutil"
	"path/filepath"
	"reflect"
	"time"
)

// renderer is used to provide a `render()` method for an `Air` instance
// for renders a "text/html" response by using `template.Template`.
type renderer struct {
	goTemplate      *template.Template
	templateFuncMap template.FuncMap
	air             *Air
}

// defaultTemplateFuncMap is a default template func map of `renderer`
var defaultTemplateFuncMap template.FuncMap

func init() {
	tfm := make(template.FuncMap)

	tfm["strlen"] = strlen
	tfm["substr"] = substr
	tfm["timefmt"] = timefmt
	tfm["eq"] = eq
	tfm["ne"] = ne
	tfm["lt"] = lt
	tfm["le"] = le
	tfm["gt"] = gt
	tfm["ge"] = ge

	defaultTemplateFuncMap = tfm
}

// newRenderer returns a new instance of `Renderer`.
func newRenderer(a *Air) *renderer {
	return &renderer{
		templateFuncMap: defaultTemplateFuncMap,
		air:             a,
	}
}

// parseTemplates parses files into templates.
//
// e.g. r.air.Config.TemplatesRoot == "templates"
//
// templates/
//   index.html
//   login.html
//   register.html
//
// templates/parts/
//   header.html
//   footer.html
//
// will be parsed into:
//
// "index.html", "login.html", "register.html",
// "parts/header.html", "parts/footer.html".
func (r *renderer) parseTemplates() {
	tr := filepath.Clean(r.air.Config.TemplatesRoot)
	filenames, err := filepath.Glob(tr + "/*/*.html")
	if err != nil {
		panic(err)
	}
	for _, filename := range filenames {
		b, err := ioutil.ReadFile(filename)
		if err != nil {
			panic(err)
		}
		name := filepath.ToSlash(filename[len(tr):])
		if name[0] == '/' {
			name = name[1:]
		}
		var tmpl *template.Template
		if r.goTemplate == nil {
			r.goTemplate = template.New(name).Funcs(r.templateFuncMap)
		}
		if name == r.goTemplate.Name() {
			tmpl = r.goTemplate
		} else {
			tmpl = r.goTemplate.New(name)
		}
		_, err = tmpl.Parse(string(b))
		if err != nil {
			panic(err)
		}
	}
}

// render renders a "text/html" response by using `template.Template`.
func (r *renderer) render(wr io.Writer, templateName string, c *Context) error {
	return r.goTemplate.ExecuteTemplate(wr, templateName, c.Data)
}

// strlen returns the number of char in the s.
func strlen(s string) int {
	return len([]rune(s))
}

// substr returns the substring consisting of the chars of the s starting
// at index i and continuing up to, but not including, the char at index j.
func substr(s string, i, j int) string {
	rs := []rune(s)
	return string(rs[i:j])
}

// timefmt returns a textual representation of the time value formatted
// according to layout.
func timefmt(t time.Time, layout string) string {
	return t.Format(layout)
}

// eq reports whether the v is equal to one of the ovs.
// It means v == v1 || v == v2 || ...
func eq(v interface{}, ovs ...interface{}) bool {
	for _, ov := range ovs {
		if ov == v {
			return true
		}
	}
	return false
}

// ne reports whether the v is not equal to any of the ovs.
// It means v != v1 && v != v2 && ...
func ne(v interface{}, ovs ...interface{}) bool {
	return !eq(v, ovs...)
}

// lt reports whether the a is less than the b.
// It means a < b.
func lt(a, b interface{}) bool {
	switch a.(type) {
	case int, int8, int16, int32, int64:
		return reflect.ValueOf(a).Int() < reflect.ValueOf(b).Int()
	case uint, uint8, uint16, uint32, uint64, uintptr:
		return reflect.ValueOf(a).Uint() < reflect.ValueOf(b).Uint()
	case float32, float64:
		return reflect.ValueOf(a).Float() < reflect.ValueOf(b).Float()
	case string:
		return a.(string) < b.(string)
	default:
		panic("invalid kind")
	}
}

// le reports whether the a is less than or equal to the b.
// It means a <= b.
func le(a, b interface{}) bool {
	return lt(a, b) || eq(a, b)
}

// gt reports whether the a is greater than the b.
// It means a > b.
func gt(a, b interface{}) bool {
	return !le(a, b)
}

// ge reports whether the a is greater than or equal to the b.
// It means a >= b.
func ge(a, b interface{}) bool {
	return lt(a, b)
}
{'fraction_non_alphanumeric': 0.15322943878929826, 'fraction_numerical': 0.011440410773804161, 'mean_word_length': 3.2601688411358403, 'pattern_counts': {'":': 0, '<': 26, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '167204', 'n_tokens_mistral': 4183, 'n_tokens_neox': 3981, 'n_words': 1174}