<commit_before>package kocha

import (
	"fmt"
	"html/template"
	"io/ioutil"
	"os"
	"path/filepath"
	"reflect"
	"strings"
	"time"
)

var (
	TemplateFuncs = template.FuncMap{
		"eq": func(a, b interface{}) bool {
			// TODO: remove in Go 1.2
			//       see http://tip.golang.org/pkg/text/template/#hdr-Functions
			return a == b
		},
		"ne": func(a, b interface{}) bool {
			// TODO: remove in Go 1.2
			//       see http://tip.golang.org/pkg/text/template/#hdr-Functions
			return a != b
		},
		"in": func(a, b interface{}) bool {
			v := reflect.ValueOf(a)
			switch v.Kind() {
			case reflect.Slice, reflect.Array, reflect.String:
				if v.IsNil() {
					return false
				}
				for i := 0; i < v.Len(); i++ {
					if v.Index(i).Interface() == b {
						return true
					}
				}
			default:
				panic(fmt.Errorf("invalid type %v: valid types are slice, array and string", v.Type().Name()))
			}
			return false
		},
		"url": Reverse,
		"nl2br": func(text string) template.HTML {
			return template.HTML(strings.Replace(template.HTMLEscapeString(text), "\n", "<br>", -1))
		},
		"raw": func(text string) template.HTML {
			return template.HTML(text)
		},
		"date": func(date time.Time, layout string) string {
			return date.Format(layout)
		},
	}
)

type TemplateSet map[string]map[string]*template.Template

func (t TemplateSet) Get(appName, name, format string) *template.Template {
	return t[appName][fmt.Sprintf("%s.%s", ToSnakeCase(name), format)]
}

func (t TemplateSet) Ident(appName, name, format string) string {
	return fmt.Sprintf("%s:%s.%s", appName, ToSnakeCase(name), format)
}

func TemplateSetFromPaths(templateSetPaths map[string][]string) TemplateSet {
	layoutPaths := make(map[string]map[string]string)
	templatePaths := make(map[string]map[string]string)
	for appName, paths := range templateSetPaths {
		layoutPaths[appName] = make(map[string]string)
		templatePaths[appName] = make(map[string]string)
		for _, rootPath := range paths {
			if err := filepath.Walk(rootPath, func(path string, info os.FileInfo, err error) error {
				if err != nil {
					return err
				}
				if info.IsDir() {
					return nil
				}
				name, err := filepath.Rel(rootPath, path)
				if err != nil {
					return err
				}
				if filepath.HasPrefix(name, "layouts/") {
					if layoutPath, ok := layoutPaths[appName][name]; ok {
						return fmt.Errorf("duplicate name of layout file:\n  1. %s\n  2. %s\n", layoutPath, path)
					}
					layoutPaths[appName][name] = path
				} else {
					if templatePath, ok := templatePaths[appName][name]; ok {
						return fmt.Errorf("duplicate name of template file:\n  1. %s\n  2. %s\n", templatePath, path)
					}
					templatePaths[appName][name] = path
				}
				return nil
			}); err != nil {
				panic(err)
			}
		}
	}
	templateSet := make(TemplateSet)
	for layoutAppName, layouts := range layoutPaths {
		templateSet[layoutAppName] = make(map[string]*template.Template)
		for layoutName, layoutPath := range layouts {
			layoutBytes, err := ioutil.ReadFile(layoutPath)
			if err != nil {
				panic(err)
			}
			layoutTemplate := template.Must(template.New("layout").Funcs(TemplateFuncs).Parse(string(layoutBytes)))
			templateSet[layoutAppName][layoutName] = layoutTemplate
			for templateAppName, templates := range templatePaths {
				templateSet[templateAppName] = make(map[string]*template.Template)
				for templateName, templatePath := range templates {
					// do not use the layoutTemplate.Clone() in order to retrieve layout as string by `kocha build`
					layout := template.Must(template.New("layout").Funcs(TemplateFuncs).Parse(string(layoutBytes)))
					t := template.Must(layout.ParseFiles(templatePath))
					templateSet[templateAppName][templateName] = t
				}
			}
		}
	}
	return templateSet
}
<commit_msg>Use filepath.ToSlash() to work on windows<commit_after>package kocha

import (
	"fmt"
	"html/template"
	"io/ioutil"
	"os"
	"path/filepath"
	"reflect"
	"strings"
	"time"
)

var (
	TemplateFuncs = template.FuncMap{
		"eq": func(a, b interface{}) bool {
			// TODO: remove in Go 1.2
			//       see http://tip.golang.org/pkg/text/template/#hdr-Functions
			return a == b
		},
		"ne": func(a, b interface{}) bool {
			// TODO: remove in Go 1.2
			//       see http://tip.golang.org/pkg/text/template/#hdr-Functions
			return a != b
		},
		"in": func(a, b interface{}) bool {
			v := reflect.ValueOf(a)
			switch v.Kind() {
			case reflect.Slice, reflect.Array, reflect.String:
				if v.IsNil() {
					return false
				}
				for i := 0; i < v.Len(); i++ {
					if v.Index(i).Interface() == b {
						return true
					}
				}
			default:
				panic(fmt.Errorf("invalid type %v: valid types are slice, array and string", v.Type().Name()))
			}
			return false
		},
		"url": Reverse,
		"nl2br": func(text string) template.HTML {
			return template.HTML(strings.Replace(template.HTMLEscapeString(text), "\n", "<br>", -1))
		},
		"raw": func(text string) template.HTML {
			return template.HTML(text)
		},
		"date": func(date time.Time, layout string) string {
			return date.Format(layout)
		},
	}
)

type TemplateSet map[string]map[string]*template.Template

func (t TemplateSet) Get(appName, name, format string) *template.Template {
	return t[appName][fmt.Sprintf("%s.%s", ToSnakeCase(name), format)]
}

func (t TemplateSet) Ident(appName, name, format string) string {
	return fmt.Sprintf("%s:%s.%s", appName, ToSnakeCase(name), format)
}

func TemplateSetFromPaths(templateSetPaths map[string][]string) TemplateSet {
	layoutPaths := make(map[string]map[string]string)
	templatePaths := make(map[string]map[string]string)
	for appName, paths := range templateSetPaths {
		layoutPaths[appName] = make(map[string]string)
		templatePaths[appName] = make(map[string]string)
		for _, rootPath := range paths {
			if err := filepath.Walk(rootPath, func(path string, info os.FileInfo, err error) error {
				if err != nil {
					return err
				}
				if info.IsDir() {
					return nil
				}
				name, err := filepath.Rel(rootPath, path)
				if err != nil {
					return err
				}
				if filepath.HasPrefix(filepath.ToSlash(name), "layouts/") {
					if layoutPath, ok := layoutPaths[appName][name]; ok {
						return fmt.Errorf("duplicate name of layout file:\n  1. %s\n  2. %s\n", layoutPath, path)
					}
					layoutPaths[appName][name] = path
				} else {
					if templatePath, ok := templatePaths[appName][name]; ok {
						return fmt.Errorf("duplicate name of template file:\n  1. %s\n  2. %s\n", templatePath, path)
					}
					templatePaths[appName][name] = path
				}
				return nil
			}); err != nil {
				panic(err)
			}
		}
	}
	templateSet := make(TemplateSet)
	for layoutAppName, layouts := range layoutPaths {
		templateSet[layoutAppName] = make(map[string]*template.Template)
		for layoutName, layoutPath := range layouts {
			layoutBytes, err := ioutil.ReadFile(layoutPath)
			if err != nil {
				panic(err)
			}
			layoutTemplate := template.Must(template.New("layout").Funcs(TemplateFuncs).Parse(string(layoutBytes)))
			templateSet[layoutAppName][layoutName] = layoutTemplate
			for templateAppName, templates := range templatePaths {
				templateSet[templateAppName] = make(map[string]*template.Template)
				for templateName, templatePath := range templates {
					// do not use the layoutTemplate.Clone() in order to retrieve layout as string by `kocha build`
					layout := template.Must(template.New("layout").Funcs(TemplateFuncs).Parse(string(layoutBytes)))
					t := template.Must(layout.ParseFiles(templatePath))
					templateSet[templateAppName][templateName] = t
				}
			}
		}
	}
	return templateSet
}

{'fraction_non_alphanumeric': 0.14556962025316456, 'fraction_numerical': 0.0029008438818565402, 'mean_word_length': 3.531063321385902, 'pattern_counts': {'":': 14, '<': 7, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2673551', 'n_tokens_mistral': 2941, 'n_tokens_neox': 2492, 'n_words': 595}