<commit_before>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Template library.  See http://code.google.com/p/json-template/wiki/Reference
// TODO: document this here as well.
package template

import (
	"fmt";
	"io";
	"os";
	"reflect";
	"strings";
	"template";
)

var ErrLBrace = os.NewError("unexpected opening brace")
var ErrUnmatchedRBrace = os.NewError("unmatched closing brace")
var ErrUnmatchedLBrace = os.NewError("unmatched opening brace")
var ErrBadDirective = os.NewError("unrecognized directive name")
var ErrEmptyDirective = os.NewError("empty directive")
var ErrFields = os.NewError("incorrect fields for directive")
var ErrSyntax = os.NewError("directive out of place")
var ErrNoEnd = os.NewError("section does not have .end")
var ErrNoVar = os.NewError("variable name not in struct");
var ErrBadType = os.NewError("unsupported type for variable");
var ErrNotStruct = os.NewError("driver must be a struct")
var ErrNoFormatter = os.NewError("unknown formatter")

// All the literals are aces.
var lbrace = []byte{ '{' }
var rbrace = []byte{ '}' }
var space = []byte{ ' ' }

// The various types of "tokens", which are plain text or brace-delimited descriptors
const (
	Alternates = iota;
	Comment;
	End;
	Literal;
	Or;
	Repeated;
	Section;
	Text;
	Variable;
)

// FormatterMap is the type describing the mapping from formatter
// names to the functions that implement them.
// TODO(rsc): Maybe func should take interface{} instead?
type FormatterMap map[string] func(reflect.Value) string

// Built-in formatters.
var builtins = FormatterMap {
	"html" : HtmlFormatter,
	"str" : StringFormatter,
	"" : StringFormatter,
}

type template struct {
	errorchan	chan *os.Error;	// for erroring out
	linenum	*int;	// shared by all templates derived from this one
	parent	*template;
	data	reflect.Value;	// the driver data for this section etc.
	fmap	FormatterMap;	// formatters for variables
	buf	[]byte;	// input text to process
	p	int;	// position in buf
	wr	io.Write;	// where to send output
}

// Create a top-level template
func newTemplate(ch chan *os.Error, linenum *int, buf []byte, data reflect.Value, fmap FormatterMap, wr io.Write) *template {
	t := new(template);
	t.errorchan = ch;
	t.linenum = linenum;
	*linenum = 1;
	t.parent = nil;
	t.data = data;
	t.buf = buf;
	t.p = 0;
	t.fmap = fmap;
	t.wr = wr;
	return t;
}

// Create a template deriving from its parent
func childTemplate(parent *template, buf []byte, data reflect.Value) *template {
	t := newTemplate(parent.errorchan, parent.linenum, buf, data, parent.fmap, parent.wr);
	t.parent = parent;
	return t;
}

// Report error and stop generation.
func (t *template) error(err *os.Error, args ...) {
	fmt.Fprintf(os.Stderr, "template error: line %d: %s%s\n", *t.linenum, err, fmt.Sprint(args));  // TODO: drop this? (only way to get line number)
	t.errorchan <- err;
	sys.Goexit();
}

func white(c uint8) bool {
	return c == ' ' || c == '\t' || c == '\n'
}

func (t *template) execute()
func (t *template) executeSection(w []string)

// nextItem returns the next item from the input buffer.  If the returned
// item is empty, we are at EOF.  The item will be either a brace-
// delimited string or a non-empty string between brace-delimited
// strings.  Most tokens stop at (but include, if plain text) a newline.
// Action tokens on a line by themselves drop the white space on
// either side, up to and including the newline.
func (t *template) nextItem() []byte {
	brace := false;	// are we waiting for an opening brace?
	special := false;	// is this a {.foo} directive, which means trim white space?
	// Delete surrounding white space if this {.foo} is the only thing on the line.
	trim_white := t.p == 0 || t.buf[t.p-1] == '\n';
	only_white := true;	// we have seen only white space so far
	var i int;
	start := t.p;
Loop:
	for i = t.p; i < len(t.buf); i++ {
		switch t.buf[i] {
		case '\n':
			*t.linenum++;
			i++;
			break Loop;
		case ' ', '\t':
			// white space, do nothing
		case '{':
			if brace {
				t.error(ErrLBrace)
			}
			// anything interesting already on the line?
			if !only_white {
				break Loop;
			}
			// is it a directive or comment?
			if i+2 < len(t.buf) && (t.buf[i+1] == '.' || t.buf[i+1] == '#') {
				special = true;<commit_msg>\r is white space<commit_after>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Template library.  See http://code.google.com/p/json-template/wiki/Reference
// TODO: document this here as well.
package template

import (
	"fmt";
	"io";
	"os";
	"reflect";
	"strings";
	"template";
)

var ErrLBrace = os.NewError("unexpected opening brace")
var ErrUnmatchedRBrace = os.NewError("unmatched closing brace")
var ErrUnmatchedLBrace = os.NewError("unmatched opening brace")
var ErrBadDirective = os.NewError("unrecognized directive name")
var ErrEmptyDirective = os.NewError("empty directive")
var ErrFields = os.NewError("incorrect fields for directive")
var ErrSyntax = os.NewError("directive out of place")
var ErrNoEnd = os.NewError("section does not have .end")
var ErrNoVar = os.NewError("variable name not in struct");
var ErrBadType = os.NewError("unsupported type for variable");
var ErrNotStruct = os.NewError("driver must be a struct")
var ErrNoFormatter = os.NewError("unknown formatter")

// All the literals are aces.
var lbrace = []byte{ '{' }
var rbrace = []byte{ '}' }
var space = []byte{ ' ' }

// The various types of "tokens", which are plain text or brace-delimited descriptors
const (
	Alternates = iota;
	Comment;
	End;
	Literal;
	Or;
	Repeated;
	Section;
	Text;
	Variable;
)

// FormatterMap is the type describing the mapping from formatter
// names to the functions that implement them.
// TODO(rsc): Maybe func should take interface{} instead?
type FormatterMap map[string] func(reflect.Value) string

// Built-in formatters.
var builtins = FormatterMap {
	"html" : HtmlFormatter,
	"str" : StringFormatter,
	"" : StringFormatter,
}

type template struct {
	errorchan	chan *os.Error;	// for erroring out
	linenum	*int;	// shared by all templates derived from this one
	parent	*template;
	data	reflect.Value;	// the driver data for this section etc.
	fmap	FormatterMap;	// formatters for variables
	buf	[]byte;	// input text to process
	p	int;	// position in buf
	wr	io.Write;	// where to send output
}

// Create a top-level template
func newTemplate(ch chan *os.Error, linenum *int, buf []byte, data reflect.Value, fmap FormatterMap, wr io.Write) *template {
	t := new(template);
	t.errorchan = ch;
	t.linenum = linenum;
	*linenum = 1;
	t.parent = nil;
	t.data = data;
	t.buf = buf;
	t.p = 0;
	t.fmap = fmap;
	t.wr = wr;
	return t;
}

// Create a template deriving from its parent
func childTemplate(parent *template, buf []byte, data reflect.Value) *template {
	t := newTemplate(parent.errorchan, parent.linenum, buf, data, parent.fmap, parent.wr);
	t.parent = parent;
	return t;
}

// Report error and stop generation.
func (t *template) error(err *os.Error, args ...) {
	fmt.Fprintf(os.Stderr, "template error: line %d: %s%s\n", *t.linenum, err, fmt.Sprint(args));  // TODO: drop this? (only way to get line number)
	t.errorchan <- err;
	sys.Goexit();
}

func white(c uint8) bool {
	return c == ' ' || c == '\t' || c == '\r' || c == '\n'
}

func (t *template) execute()
func (t *template) executeSection(w []string)

// nextItem returns the next item from the input buffer.  If the returned
// item is empty, we are at EOF.  The item will be either a brace-
// delimited string or a non-empty string between brace-delimited
// strings.  Most tokens stop at (but include, if plain text) a newline.
// Action tokens on a line by themselves drop the white space on
// either side, up to and including the newline.
func (t *template) nextItem() []byte {
	brace := false;	// are we waiting for an opening brace?
	special := false;	// is this a {.foo} directive, which means trim white space?
	// Delete surrounding white space if this {.foo} is the only thing on the line.
	trim_white := t.p == 0 || t.buf[t.p-1] == '\n';
	only_white := true;	// we have seen only white space so far
	var i int;
	start := t.p;
Loop:
	for i = t.p; i < len(t.buf); i++ {
		switch t.buf[i] {
		case '\n':
			*t.linenum++;
			i++;
			break Loop;
		case ' ', '\t', '\r':
			// white space, do nothing
		case '{':
			if brace {
				t.error(ErrLBrace)
			}
			// anything interesting already on the line?
			if !only_white {
				break Loop;
			}
			// is it a directive or comment?
			if i+2 < len(t.buf) && (t.buf[i+1] == '.' || t.buf[i+1] == '#') {
				special = true;
{'fraction_non_alphanumeric': 0.13466162830642392, 'fraction_numerical': 0.0027481964960494676, 'mean_word_length': 3.553701772679875, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4712018', 'n_tokens_mistral': 3017, 'n_tokens_neox': 2893, 'n_words': 1071}