<commit_before>package template

import (
	"bytes"
	"fmt"
	"gnd.la/util/types"
	"html/template"
	"reflect"
	"runtime"
	"strings"
	"text/template/parse"
)

var (
	errType          = reflect.TypeOf((*error)(nil)).Elem()
	stringType       = reflect.TypeOf("")
	htmlType         = reflect.TypeOf(HTML(""))
	templateHtmlType = reflect.TypeOf(template.HTML(""))
	jsType           = reflect.TypeOf(JS(""))
	templateJsType   = reflect.TypeOf(template.JS(""))
	stringerType     = reflect.TypeOf((*fmt.Stringer)(nil)).Elem()
	emptyType        = reflect.TypeOf((*interface{})(nil)).Elem()
	zero             = reflect.Zero(emptyType)
)

const (
	templateHtmlEscaper = "html_template_htmlescaper"
	templateJsEscaper   = "html_template_jsvalescaper"
)

// TODO: Remove variables inside if or with when exiting the scope

type opcode uint8

const (
	opNOP opcode = iota
	opFIELD
	opFUNC
	opITER
	opJMP
	opJMPF
	opJMPT
	opMARK
	opNEXT
	opDOT
	opPRINT
	opPUSHDOT
	opPOPDOT
	opPOP
	opSETVAR
	opSTRING
	opTEMPLATE
	opVAL
	opVAR
	opWB
)

type valType uint32

type inst struct {
	op  opcode
	val valType
}

func encodeVal(high int, low valType) valType {
	return valType(high<<16) | low
}

func decodeVal(val valType) (int, int) {
	return int(val >> 16), int(val & 0xFFFF)
}

type iterator interface {
	Next() (bool, reflect.Value, reflect.Value)
}

type nilIterator struct {
}

func (it *nilIterator) Next() (bool, reflect.Value, reflect.Value) {
	return false, zero, zero
}

type sliceIterator struct {
	val    reflect.Value
	pos    int
	length int
}

func (it *sliceIterator) Next() (bool, reflect.Value, reflect.Value) {
	if it.pos < it.length {
		val := it.val.Index(it.pos)
		pos := reflect.ValueOf(it.pos)
		it.pos++
		return true, pos, val
	}
	return false, zero, zero
}

type mapIterator struct {
	val  reflect.Value
	keys []reflect.Value
	pos  int
}

func (it *mapIterator) Next() (bool, reflect.Value, reflect.Value) {
	if it.pos < len(it.keys) {
		k := it.keys[it.pos]
		val := it.val.MapIndex(k)
		it.pos++
		return true, k, val
	}
	return false, zero, zero
}

type chanIterator struct {
	val reflect.Value
	pos int
}

func (it *chanIterator) Next() (bool, reflect.Value, reflect.Value) {
	val, ok := it.val.Recv()
	if ok {
		pos := reflect.ValueOf(it.pos)
		it.pos++
		return true, pos, val
	}
	return false, zero, zero
}

func newIterator(v reflect.Value) (iterator, error) {
	for v.Kind() == reflect.Ptr || v.Kind() == reflect.Interface {
		if v.IsNil() {
			return &nilIterator{}, nil
		}
	}
	switch v.Kind() {
	case reflect.Slice, reflect.Array:
		if v.Len() == 0 {
			return &nilIterator{}, nil
		}
		return &sliceIterator{val: v, length: v.Len()}, nil
	case reflect.Map:
		if v.Len() == 0 {
			return &nilIterator{}, nil
		}
		keys := v.MapKeys()
		types.SortValues(keys)
		return &mapIterator{val: v, keys: keys}, nil
	case reflect.Chan:
		if v.IsNil() {
			return &nilIterator{}, nil
		}
		return &chanIterator{val: v}, nil
	case reflect.Invalid:
		return &nilIterator{}, nil
	}
	return nil, fmt.Errorf("can't range over %T", v.Interface())
}

type variable struct {
	name  string
	value reflect.Value
}

var statePool = make(chan *state, runtime.GOMAXPROCS(0))

type state struct {
	p     *program
	w     *bytes.Buffer
	vars  []variable
	stack []reflect.Value
	marks []int
	dot   []reflect.Value
}

func newState(p *program, w *bytes.Buffer) *state {
	select {
	case s := <-statePool:
		s.reset()
		s.p = p
		s.w = w
		return s
	default:
		return &state{
			p: p,
			w: w,
		}
	}
}

func (s *state) reset() {
	s.vars = s.vars[:0]
	s.stack = s.stack[:0]
	s.marks = s.marks[:0]
	s.dot = s.dot[:0]
}

func (s *state) put() {
	select {
	case statePool <- s:
	default:
	}
}

func (s *state) formatErr(pc int, tmpl string, err error) error {
	if p := strings.Index(tmpl, "$htmltemplate"); p >= 0 {
		// This is a mangled tree generated by html/template,
		// which has not text. Use the unmangled version instead.
		tmpl = tmpl[:p-1]
	}
	tr := s.p.tmpl.trees[tmpl]
	if tr != nil {
		ctx := s.p.context[tmpl]
		for ii, v := range ctx {
			if v.pc > pc {
				if ii > 0 {
					node := ctx[ii-1].node
					loc, _ := tr.ErrorContext(node)
					if loc != "" {
						// Need to check if the error was in line 1, to remove
						// the length of the text we prepend to templates.
						// TODO: Actually, that's prepended after every define,
						// eventually fix that too.
						p := strings.SplitN(loc, ":", 2)
						if len(p) == 2 {
							var line int
							var pos int
							if _, err := fmt.Sscanf(p[1], "%d:%d", &line, &pos); err == nil {
								if line == 1 {
									pos -= len(templatePrepend)
								}
								loc = fmt.Sprintf("%s:%d:%d", p[0], line, pos)
							}
						}
						return fmt.Errorf("%s: %s", loc, err.Error())
					}
				}
				break
			}
		}
	}
	return err
}

func (s *state) errorf(pc int, tmpl string, format string, args ...interface{}) error {
	err := fmt.Errorf(format, args...)
	return s.formatErr(pc, tmpl, err)
}

func (s *state) pushVar(name string, value reflect.Value) {
	s.vars = append(s.vars, variable{name, value})
}

func (s *state) varMark() int {
	return len(s.vars)
}

func (s *state) popVar(mark int) {
	s.vars = s.vars[0:mark]
}

func (s *state) setVar(n int, value reflect.Value) {
	s.vars[len(s.vars)-n].value = value
}

func (s *state) varValue(name string) (reflect.Value, error) {
	for i := s.varMark() - 1; i >= 0; i-- {
		if s.vars[i].name == name {
			return s.vars[i].value, nil
		}
	}
	return reflect.Value{}, fmt.Errorf("undefined variable: %q", name)
}

// call fn, remove its args from the stack and push the result
func (s *state) call(fn reflect.Value, name string, args int) error {
	pos := len(s.stack) - args
	in := s.stack[pos:]
	// arguments are in reverse order
	for ii := 0; ii < len(in)/2; ii++ {
		in[ii], in[len(in)-1-ii] = in[len(in)-1-ii], in[ii]
	}
	res := fn.Call(in)
	if len(res) == 2 && !res[1].IsNil() {
		return fmt.Errorf("error calling %q: %s", name, res[1].Interface())
	}
	s.stack = append(s.stack[:pos], stackable(res[0]))
	return nil
}

func (s *state) recover(pc *int, tmpl *string, err *error) {
	if r := recover(); r != nil {
		e, ok := r.(error)
		if !ok {
			e = fmt.Errorf("%v", r)
		}
		// TODO: Check if we can find if we were calling
		// a function, either from a opFUNC or opFIELD.
		*err = s.formatErr(*pc, *tmpl, e)
	}
}

func (s *state) execute(tmpl string, ns string, dot reflect.Value) (err error) {
	code, ok := s.p.code[tmpl]
	if !ok {
		return fmt.Errorf("template %q does not exist", tmpl)
	}
	s.pushVar("", dot)
	if ns != "" {
		if vars, err := s.varValue("Vars"); err == nil {
			if !vars.IsNil() {
				s.pushVar("Vars", reflect.ValueOf(vars.Interface().(VarMap).Unpack(ns)))<commit_msg>Don't check for undefined templates at execution time<commit_after>package template

import (
	"bytes"
	"fmt"
	"gnd.la/util/types"
	"html/template"
	"reflect"
	"runtime"
	"strings"
	"text/template/parse"
)

var (
	errType          = reflect.TypeOf((*error)(nil)).Elem()
	stringType       = reflect.TypeOf("")
	htmlType         = reflect.TypeOf(HTML(""))
	templateHtmlType = reflect.TypeOf(template.HTML(""))
	jsType           = reflect.TypeOf(JS(""))
	templateJsType   = reflect.TypeOf(template.JS(""))
	stringerType     = reflect.TypeOf((*fmt.Stringer)(nil)).Elem()
	emptyType        = reflect.TypeOf((*interface{})(nil)).Elem()
	zero             = reflect.Zero(emptyType)
)

const (
	templateHtmlEscaper = "html_template_htmlescaper"
	templateJsEscaper   = "html_template_jsvalescaper"
)

// TODO: Remove variables inside if or with when exiting the scope

type opcode uint8

const (
	opNOP opcode = iota
	opFIELD
	opFUNC
	opITER
	opJMP
	opJMPF
	opJMPT
	opMARK
	opNEXT
	opDOT
	opPRINT
	opPUSHDOT
	opPOPDOT
	opPOP
	opSETVAR
	opSTRING
	opTEMPLATE
	opVAL
	opVAR
	opWB
)

type valType uint32

type inst struct {
	op  opcode
	val valType
}

func encodeVal(high int, low valType) valType {
	return valType(high<<16) | low
}

func decodeVal(val valType) (int, int) {
	return int(val >> 16), int(val & 0xFFFF)
}

type iterator interface {
	Next() (bool, reflect.Value, reflect.Value)
}

type nilIterator struct {
}

func (it *nilIterator) Next() (bool, reflect.Value, reflect.Value) {
	return false, zero, zero
}

type sliceIterator struct {
	val    reflect.Value
	pos    int
	length int
}

func (it *sliceIterator) Next() (bool, reflect.Value, reflect.Value) {
	if it.pos < it.length {
		val := it.val.Index(it.pos)
		pos := reflect.ValueOf(it.pos)
		it.pos++
		return true, pos, val
	}
	return false, zero, zero
}

type mapIterator struct {
	val  reflect.Value
	keys []reflect.Value
	pos  int
}

func (it *mapIterator) Next() (bool, reflect.Value, reflect.Value) {
	if it.pos < len(it.keys) {
		k := it.keys[it.pos]
		val := it.val.MapIndex(k)
		it.pos++
		return true, k, val
	}
	return false, zero, zero
}

type chanIterator struct {
	val reflect.Value
	pos int
}

func (it *chanIterator) Next() (bool, reflect.Value, reflect.Value) {
	val, ok := it.val.Recv()
	if ok {
		pos := reflect.ValueOf(it.pos)
		it.pos++
		return true, pos, val
	}
	return false, zero, zero
}

func newIterator(v reflect.Value) (iterator, error) {
	for v.Kind() == reflect.Ptr || v.Kind() == reflect.Interface {
		if v.IsNil() {
			return &nilIterator{}, nil
		}
	}
	switch v.Kind() {
	case reflect.Slice, reflect.Array:
		if v.Len() == 0 {
			return &nilIterator{}, nil
		}
		return &sliceIterator{val: v, length: v.Len()}, nil
	case reflect.Map:
		if v.Len() == 0 {
			return &nilIterator{}, nil
		}
		keys := v.MapKeys()
		types.SortValues(keys)
		return &mapIterator{val: v, keys: keys}, nil
	case reflect.Chan:
		if v.IsNil() {
			return &nilIterator{}, nil
		}
		return &chanIterator{val: v}, nil
	case reflect.Invalid:
		return &nilIterator{}, nil
	}
	return nil, fmt.Errorf("can't range over %T", v.Interface())
}

type variable struct {
	name  string
	value reflect.Value
}

var statePool = make(chan *state, runtime.GOMAXPROCS(0))

type state struct {
	p     *program
	w     *bytes.Buffer
	vars  []variable
	stack []reflect.Value
	marks []int
	dot   []reflect.Value
}

func newState(p *program, w *bytes.Buffer) *state {
	select {
	case s := <-statePool:
		s.reset()
		s.p = p
		s.w = w
		return s
	default:
		return &state{
			p: p,
			w: w,
		}
	}
}

func (s *state) reset() {
	s.vars = s.vars[:0]
	s.stack = s.stack[:0]
	s.marks = s.marks[:0]
	s.dot = s.dot[:0]
}

func (s *state) put() {
	select {
	case statePool <- s:
	default:
	}
}

func (s *state) formatErr(pc int, tmpl string, err error) error {
	if p := strings.Index(tmpl, "$htmltemplate"); p >= 0 {
		// This is a mangled tree generated by html/template,
		// which has not text. Use the unmangled version instead.
		tmpl = tmpl[:p-1]
	}
	tr := s.p.tmpl.trees[tmpl]
	if tr != nil {
		ctx := s.p.context[tmpl]
		for ii, v := range ctx {
			if v.pc > pc {
				if ii > 0 {
					node := ctx[ii-1].node
					loc, _ := tr.ErrorContext(node)
					if loc != "" {
						// Need to check if the error was in line 1, to remove
						// the length of the text we prepend to templates.
						// TODO: Actually, that's prepended after every define,
						// eventually fix that too.
						p := strings.SplitN(loc, ":", 2)
						if len(p) == 2 {
							var line int
							var pos int
							if _, err := fmt.Sscanf(p[1], "%d:%d", &line, &pos); err == nil {
								if line == 1 {
									pos -= len(templatePrepend)
								}
								loc = fmt.Sprintf("%s:%d:%d", p[0], line, pos)
							}
						}
						return fmt.Errorf("%s: %s", loc, err.Error())
					}
				}
				break
			}
		}
	}
	return err
}

func (s *state) errorf(pc int, tmpl string, format string, args ...interface{}) error {
	err := fmt.Errorf(format, args...)
	return s.formatErr(pc, tmpl, err)
}

func (s *state) pushVar(name string, value reflect.Value) {
	s.vars = append(s.vars, variable{name, value})
}

func (s *state) varMark() int {
	return len(s.vars)
}

func (s *state) popVar(mark int) {
	s.vars = s.vars[0:mark]
}

func (s *state) setVar(n int, value reflect.Value) {
	s.vars[len(s.vars)-n].value = value
}

func (s *state) varValue(name string) (reflect.Value, error) {
	for i := s.varMark() - 1; i >= 0; i-- {
		if s.vars[i].name == name {
			return s.vars[i].value, nil
		}
	}
	return reflect.Value{}, fmt.Errorf("undefined variable: %q", name)
}

// call fn, remove its args from the stack and push the result
func (s *state) call(fn reflect.Value, name string, args int) error {
	pos := len(s.stack) - args
	in := s.stack[pos:]
	// arguments are in reverse order
	for ii := 0; ii < len(in)/2; ii++ {
		in[ii], in[len(in)-1-ii] = in[len(in)-1-ii], in[ii]
	}
	res := fn.Call(in)
	if len(res) == 2 && !res[1].IsNil() {
		return fmt.Errorf("error calling %q: %s", name, res[1].Interface())
	}
	s.stack = append(s.stack[:pos], stackable(res[0]))
	return nil
}

func (s *state) recover(pc *int, tmpl *string, err *error) {
	if r := recover(); r != nil {
		e, ok := r.(error)
		if !ok {
			e = fmt.Errorf("%v", r)
		}
		// TODO: Check if we can find if we were calling
		// a function, either from a opFUNC or opFIELD.
		*err = s.formatErr(*pc, *tmpl, e)
	}
}

func (s *state) execute(tmpl string, ns string, dot reflect.Value) (err error) {
	code := s.p.code[tmpl]
	s.pushVar("", dot)
	if ns != "" {
		if vars, err := s.varValue("Vars"); err == nil {
			if !vars.IsNil() {
				s.pushVar("Vars", reflect.ValueOf(vars.Interface().(VarMap).Unpack(ns)))
{'fraction_non_alphanumeric': 0.16031163383024946, 'fraction_numerical': 0.0053936624466252156, 'mean_word_length': 3.068881438585797, 'pattern_counts': {'":': 2, '<': 17, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1265133', 'n_tokens_mistral': 5497, 'n_tokens_neox': 5064, 'n_words': 1414}