<commit_before>package triton

import (
	"bufio"
	"errors"
	"fmt"
	"gopkg.in/fsnotify.v1"
	"html/template"
	"io"
	"net/http"
	"os"
	"path/filepath"
	"strings"
)

// Server provides static cached web hosting. It searches the current working directory of
// execution recursively for .tmpl files containing valid html/templates. It also caches assets
// whose file extension matches the extensions specified by the client using triton. It watches
// the file system for changes and updates the static site accordingly without restarting.
//
// Template files provide two different, but simple, behaviors depending whether they are located
// in dot-directories.
//
// If a .tmpl file is within a dot directory, it is parsed and added to the master static Template
// for use by other templates.
//
// If a .tmpl file is not within a dot directory, it is parsed and added to the master static
// Template. Additionally, a URL entry to that file location and template filename (without the
// extension) is created for web clients to visit. The template executed to create that particular
// page will be the template filename (again, without extension). A quick example:
//
// If the Server was executed at <ROOT>, hosting at SITE.com, and a template file is located at
// <ROOT>/foo/bar/baz.tmpl, then a web client can go to SITE.com/foo/bar/baz.tmpl and the server
// will execute the template named "baz" and serve it.
//
// The special case is the file "#.tmpl", which will serve and match the directory it is contained
// in. So if <ROOT> hosting SITE.com has <ROOT>/#.tmpl, then the client can go to SITE.com (the
// home page!) and be served template "/".
//
// Static asset files behave as expected with the caveat that they must not be located in any
// dot-directories.
type Server struct {
	// WebHost is used to handle http requests. Do not call
	// the WebHost's ListenAndServe nor ListenAndServeTLS functions. Call
	// the triton Server's variants instead for proper setup to occur.
	//
	// If only http.Server had a function for setting its http.Handler in
	// addition to its exported field, then an interface could be used here
	// instead.
	WebHost *http.Server
	// ErrChan can be provided by the client, or is created after the
	// server starts listening for requests. The client must listen to
	// the channel for errors, and a closed channel indicates that the server is
	// no longer properly updating its content and must be restarted.
	ErrChan chan error
	// AssetFileExtensions is a map of file extensions to treat as static
	// assets when encountered in non-dot-directories. The values of the
	// map are the MIME type. This can be nil.
	AssetFileExtensionsToMIME map[string]string
	// Map between relative URL paths and template names to execute.
	staticTemplates map[string]string
	// Cached static assets.
	staticAssets map[string][]byte
	// Cached static HTML content.
	templates *template.Template
}

// initializeContent maps all content to URI locations.
func (s *Server) initializeContent() error {
	s.staticTemplates = make(map[string]string)
	s.staticAssets = make(map[string][]byte)
	s.templates = nil
	pwd, err := os.Getwd()
	if err != nil {
		return err
	}
	var cw *contentWalker
	if s.AssetFileExtensionsToMIME == nil || len(s.AssetFileExtensionsToMIME) == 0 {
		cw = NewContentWalker(".tmpl")
	} else {
		allFiles := make([]string, 0, len(s.AssetFileExtensionsToMIME)+1)
		allFiles = append(allFiles, ".tmpl")
		for k, _ := range s.AssetFileExtensionsToMIME {
			allFiles = append(allFiles, k)
		}
		cw = NewContentWalker(allFiles...)
	}
	if err = filepath.Walk(pwd, cw.Walk); err != nil {
		return err
	}
	hiddenTmplFiles, okHidden := cw.HiddenFiles(".tmpl")
	tmplFiles, ok := cw.Files(".tmpl")
	if !ok {
		return errors.New(fmt.Sprintf("No content walker .tmpl files"))
	}
	if okHidden && len(hiddenTmplFiles) > 0 {
		s.templates, err = template.ParseFiles(hiddenTmplFiles...)
		if err != nil {
			return err
		}
		s.templates, err = s.templates.ParseFiles(tmplFiles...)
	} else {
		s.templates, err = template.ParseFiles(tmplFiles...)
	}
	if err != nil {
		return err
	}
	for _, file := range tmplFiles {
		rel, err := filepath.Rel(pwd, file)
		if err != nil {
			return err
		}
		baseWithExt := filepath.Base(file)
		base := baseWithExt
		relNoExt := rel
		if filepath.Ext(file) != "" {
			base = baseWithExt[:strings.LastIndex(baseWithExt, ".")]
			relNoExt = rel[:strings.LastIndex(rel, ".")]
			if base == "#" {
				base = "/" + relNoExt[:len(relNoExt)-1]
				relNoExt = relNoExt[:len(relNoExt)-1]
			}
		}
		s.staticTemplates[base] = "/" + relNoExt
	}
	if s.AssetFileExtensionsToMIME != nil && len(s.AssetFileExtensionsToMIME) > 0 {
		for assetExt, _ := range s.AssetFileExtensionsToMIME {
			assetFiles, _ := cw.Files(assetExt)
			for _, file := range assetFiles {
				err = s.readStaticAssetsFile(pwd, file)
				if err != nil {
					return err
				}
			}
		}
	}
	return nil
}

// readStaticAssetsFile associates a specific asset file path to its content.
func (s *Server) readStaticAssetsFile(baseDir string, assetFile string) error {
	file, err := os.Open(assetFile)
	if err != nil {
		return err
	}
	defer file.Close()
	var asset []byte
	buffer := make([]byte, 1024)
	r := bufio.NewReader(file)
	for {
		n, err := r.Read(buffer)
		if err != nil && err != io.EOF {<commit_msg>Fix issue #3: Use fully qualified template name. Also defined magic constants.<commit_after>package triton

import (
	"bufio"
	"errors"
	"fmt"
	"gopkg.in/fsnotify.v1"
	"html/template"
	"io"
	"net/http"
	"os"
	"path/filepath"
	"strings"
)

const (
	rootTemplateFile = "#"
	templateFileExt  = ".tmpl"
)

// Server provides static cached web hosting. It searches the current working directory of
// execution recursively for .tmpl files containing valid html/templates. It also caches assets
// whose file extension matches the extensions specified by the client using triton. It watches
// the file system for changes and updates the static site accordingly without restarting.
//
// Template files provide two different, but simple, behaviors depending whether they are located
// in dot-directories.
//
// If a .tmpl file is within a dot directory, it is parsed and added to the master static Template
// for use by other templates.
//
// If a .tmpl file is not within a dot directory, it is parsed and added to the master static
// Template. Additionally, a URL entry to that file location and template filename (without the
// extension) is created for web clients to visit. The template executed to create that particular
// page will be the template filename (again, without extension). A quick example:
//
// If the Server was executed at <ROOT>, hosting at SITE.com, and a template file is located at
// <ROOT>/foo/bar/baz.tmpl, then a web client can go to SITE.com/foo/bar/baz.tmpl and the server
// will execute the template named "baz" and serve it.
//
// The special case is the file "#.tmpl", which will serve and match the directory it is contained
// in. So if <ROOT> hosting SITE.com has <ROOT>/#.tmpl, then the client can go to SITE.com (the
// home page!) and be served template "/".
//
// Static asset files behave as expected with the caveat that they must not be located in any
// dot-directories.
type Server struct {
	// WebHost is used to handle http requests. Do not call
	// the WebHost's ListenAndServe nor ListenAndServeTLS functions. Call
	// the triton Server's variants instead for proper setup to occur.
	//
	// If only http.Server had a function for setting its http.Handler in
	// addition to its exported field, then an interface could be used here
	// instead.
	WebHost *http.Server
	// ErrChan can be provided by the client, or is created after the
	// server starts listening for requests. The client must listen to
	// the channel for errors, and a closed channel indicates that the server is
	// no longer properly updating its content and must be restarted.
	ErrChan chan error
	// AssetFileExtensions is a map of file extensions to treat as static
	// assets when encountered in non-dot-directories. The values of the
	// map are the MIME type. This can be nil.
	AssetFileExtensionsToMIME map[string]string
	// Map between relative URL paths and template names to execute.
	staticTemplates map[string]string
	// Cached static assets.
	staticAssets map[string][]byte
	// Cached static HTML content.
	templates *template.Template
}

// initializeContent maps all content to URI locations.
func (s *Server) initializeContent() error {
	s.staticTemplates = make(map[string]string)
	s.staticAssets = make(map[string][]byte)
	s.templates = nil
	pwd, err := os.Getwd()
	if err != nil {
		return err
	}
	var cw *contentWalker
	if s.AssetFileExtensionsToMIME == nil || len(s.AssetFileExtensionsToMIME) == 0 {
		cw = newContentWalker(templateFileExt)
	} else {
		allFiles := make([]string, 0, len(s.AssetFileExtensionsToMIME)+1)
		allFiles = append(allFiles, templateFileExt)
		for k, _ := range s.AssetFileExtensionsToMIME {
			allFiles = append(allFiles, k)
		}
		cw = newContentWalker(allFiles...)
	}
	if err = filepath.Walk(pwd, cw.Walk); err != nil {
		return err
	}
	hiddenTmplFiles, okHidden := cw.HiddenFiles(templateFileExt)
	tmplFiles, ok := cw.Files(templateFileExt)
	if !ok {
		return errors.New(fmt.Sprintf("No content walker %s files", templateFileExt))
	}
	if okHidden && len(hiddenTmplFiles) > 0 {
		s.templates, err = template.ParseFiles(hiddenTmplFiles...)
		if err != nil {
			return err
		}
		s.templates, err = s.templates.ParseFiles(tmplFiles...)
	} else {
		s.templates, err = template.ParseFiles(tmplFiles...)
	}
	if err != nil {
		return err
	}
	for _, file := range tmplFiles {
		rel, err := filepath.Rel(pwd, file)
		if err != nil {
			return err
		}
		relNoExt := rel
		if filepath.Ext(file) != "" {
			relNoExt = rel[:strings.LastIndex(rel, ".")]
			baseWithExt := filepath.Base(file)
			if baseWithExt[:strings.LastIndex(baseWithExt, ".")] == rootTemplateFile {
				relNoExt = relNoExt[:len(relNoExt)-1]
			}
		}
		s.staticTemplates["/"+relNoExt] = "/" + relNoExt
	}
	if s.AssetFileExtensionsToMIME != nil && len(s.AssetFileExtensionsToMIME) > 0 {
		for assetExt, _ := range s.AssetFileExtensionsToMIME {
			assetFiles, _ := cw.Files(assetExt)
			for _, file := range assetFiles {
				err = s.readStaticAssetsFile(pwd, file)
				if err != nil {
					return err
				}
			}
		}
	}
	return nil
}

// readStaticAssetsFile associates a specific asset file path to its content.
func (s *Server) readStaticAssetsFile(baseDir string, assetFile string) error {
	file, err := os.Open(assetFile)
	if err != nil {
		return err
	}
	defer file.Close()
	var asset []byte
	buffer := make([]byte, 1024)
	r := bufio.NewReader(file)
	for {
		n, err := r.Read(buffer)
		if err != nil && err != io.EOF {
{'fraction_non_alphanumeric': 0.10641656606927291, 'fraction_numerical': 0.002228619184696815, 'mean_word_length': 3.5829787234042554, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6011600', 'n_tokens_mistral': 3499, 'n_tokens_neox': 3301, 'n_words': 1249}