<commit_before>////////////////////////////////////////////////////////////////////////////
// Package: easygen
// Purpose: Easy to use universal code/text generator
// Authors: <NAME> (c) 2015-17, All rights reserved
////////////////////////////////////////////////////////////////////////////

/*

Package easygen is an easy to use universal code/text generator library.

It can be used as a text or html generator for arbitrary purposes with arbitrary data and templates.

It can be used as a code generator, or anything that is structurally repetitive. Some command line parameter handling code generator are provided as examples, including the Go's built-in flag package, and the viper & cobra package.

Many examples have been provided to showcase its functionality, and different ways to use it.

*/
package easygen

import (
	"bytes"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"path/filepath"
	"strings"
	tt "text/template"

	"gopkg.in/yaml.v2"
)

////////////////////////////////////////////////////////////////////////////
// Constant and data type/structure definitions

////////////////////////////////////////////////////////////////////////////
// Global variables definitions

// Opts holds the actual values from the command line parameters
var Opts = Options{ExtYaml: ".yaml", ExtTmpl: ".tmpl"}

////////////////////////////////////////////////////////////////////////////
// Function definitions

// Generate2 will produce output according to the given template and driving data files,
// specified via fileNameTempl and fileNames (for data) respectively.
func Generate2(HTML bool, fileNameTempl string, fileNames ...string) (ret string) {
	Opts.TemplateFile = fileNameTempl

	for _, fileName := range fileNames {
		ret += Generate(HTML, fileName)
	}
	Opts.TemplateFile = ""
	return ret
}

// Generate0 will produce output according from driving data without a template file, using the string from strTempl as the template
func Generate0(HTML bool, strTempl string, fileName string) string {
	Opts.TemplateStr = strTempl
	ret := Generate(HTML, fileName)
	Opts.TemplateStr = ""
	return ret
}

// Generate will produce output from the template according to the corresponding driving data, fileName is for both template and data file name
func Generate(HTML bool, fileName string) string {
	var templates []string

	// Allow to use fileName with and without the @Opts.ExtYaml suffix.
	// If both <file>.yaml and <file> exist, prefer <file>.yaml.
	if _, err := os.Stat(fileName + Opts.ExtYaml); err == nil {
		fileName += Opts.ExtYaml
	} else if _, err = os.Stat(fileName); os.IsNotExist(err) {
		fileName += Opts.ExtYaml
	}

	// Allow to use @Opts.TemplateFile without the @Opts.ExtTmpl suffix.
	if len(Opts.TemplateFile) > 0 {
		for _, template := range strings.Split(Opts.TemplateFile, ",") {
			if _, err := os.Stat(template); os.IsNotExist(err) {
				template += Opts.ExtTmpl
			}
			templates = append(templates, template)
		}
	} else if idx := strings.LastIndex(fileName, "."); idx > 0 {
		templates = []string{fileName[:idx] + Opts.ExtTmpl}
	} else {
		templates = []string{fileName + Opts.ExtTmpl}
	}

	source, err := ioutil.ReadFile(fileName)
	checkError(err)

	m := make(map[interface{}]interface{})

	err = yaml.Unmarshal(source, &m)
	checkError(err)

	env := make(map[string]string)
	for _, e := range os.Environ() {
		sep := strings.Index(e, "=")
		env[e[0:sep]] = e[sep+1:]
	}
	m["ENV"] = env
	//fmt.Printf("] %+v\n", m)

	buf := new(bytes.Buffer)
	for _, template := range templates {
		t, err := ParseFiles(HTML, template)
		checkError(err)

		err = t.Execute(buf, m)
		checkError(err)
	}
	return buf.String()
}

// ParseFiles wraps parsing text or HTML template files into a single
// function, dictated by the first parameter "HTML".
// By <NAME> <EMAIL>
func ParseFiles(HTML bool, filenames ...string) (Template, error) {
	var tname string

	if len(Opts.TemplateStr) > 0 {
		tname = "TT"
	} else if len(filenames) == 0 {
		return nil, fmt.Errorf("ParseFiles called without template filename")
	} else {
		tname = filepath.Base(filenames[0])
	}

	textTemplate := tt.New(tname).Funcs(tt.FuncMap{
		"eqf":      strings.EqualFold,
		"split":    strings.Fields,
		"minus1":   minus1,
		"dateI":    dateI,
		"year4":    year4,
		"replace":  replace,
		"replacec": replacec,
	})

	if len(Opts.TemplateStr) > 0 {
		return textTemplate.Parse(Opts.TemplateStr)
	}
	return textTemplate.ParseFiles(filenames...)
}

////////////////////////////////////////////////////////////////////////////
// Version 2 Function definitions

// Process will process the standard easygen input: the `fileName` is for both template and data file names, and produce output from the template according to the corresponding driving data.
func Process(t Template, wr io.Writer, fileName string) error {
	return Process2(t, wr, fileName, fileName)
}

// Process2 will process the case that both template and data file names are given, and produce output according to the given template and driving data files,
// specified via fileNameTempl and fileNames (for data) respectively.
func Process2(t Template, wr io.Writer, fileNameTempl string, fileNames ...string) error {
	fileName := fileNames[0]

	source, err := ioutil.ReadFile(fileName + Opts.ExtYaml)
	checkError(err)

	m := make(map[interface{}]interface{})

	err = yaml.Unmarshal(source, &m)
	checkError(err)

	env := make(map[string]string)
	for _, e := range os.Environ() {
		sep := strings.Index(e, "=")
		env[e[0:sep]] = e[sep+1:]
	}
	m["ENV"] = env
	//fmt.Printf("] %+v\n", m)

	// template file name
	fileNameT := fileNameTempl
	if len(Opts.TemplateFile) > 0 {
		fileNameT = Opts.TemplateFile
	}
	fileNameT = fileNameT + Opts.ExtTmpl
	tn, err := t.ParseFiles(fileNameT)
	checkError(err)

	return tn.ExecuteTemplate(wr, filepath.Base(fileNameT), m)
}
<commit_msg>- [*] refactor easygen.go, #1<commit_after>////////////////////////////////////////////////////////////////////////////
// Package: easygen
// Purpose: Easy to use universal code/text generator
// Authors: <NAME> (c) 2015-17, All rights reserved
////////////////////////////////////////////////////////////////////////////

/*

Package easygen is an easy to use universal code/text generator library.

It can be used as a text or html generator for arbitrary purposes with arbitrary data and templates.

It can be used as a code generator, or anything that is structurally repetitive. Some command line parameter handling code generator are provided as examples, including the Go's built-in flag package, and the viper & cobra package.

Many examples have been provided to showcase its functionality, and different ways to use it.

*/
package easygen

import (
	"bytes"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"path/filepath"
	"strings"
	tt "text/template"

	"gopkg.in/yaml.v2"
)

////////////////////////////////////////////////////////////////////////////
// Constant and data type/structure definitions

////////////////////////////////////////////////////////////////////////////
// Global variables definitions

// EgData -- EasyGen driven Data
type EgData map[interface{}]interface{}

// Opts holds the actual values from the command line parameters
var Opts = Options{ExtYaml: ".yaml", ExtTmpl: ".tmpl"}

////////////////////////////////////////////////////////////////////////////
// Function definitions

// Generate2 will produce output according to the given template and driving data files,
// specified via fileNameTempl and fileNames (for data) respectively.
func Generate2(HTML bool, fileNameTempl string, fileNames ...string) (ret string) {
	Opts.TemplateFile = fileNameTempl

	for _, fileName := range fileNames {
		ret += Generate(HTML, fileName)
	}
	Opts.TemplateFile = ""
	return ret
}

// Generate0 will produce output according from driving data without a template file, using the string from strTempl as the template
func Generate0(HTML bool, strTempl string, fileName string) string {
	Opts.TemplateStr = strTempl
	ret := Generate(HTML, fileName)
	Opts.TemplateStr = ""
	return ret
}

// Generate will produce output from the template according to the corresponding driving data, fileName is for both template and data file name
func Generate(HTML bool, fileName string) string {
	var templates []string

	// Allow to use fileName with and without the @Opts.ExtYaml suffix.
	// If both <file>.yaml and <file> exist, prefer <file>.yaml.
	if _, err := os.Stat(fileName + Opts.ExtYaml); err == nil {
		fileName += Opts.ExtYaml
	} else if _, err = os.Stat(fileName); os.IsNotExist(err) {
		fileName += Opts.ExtYaml
	}

	// Allow to use @Opts.TemplateFile without the @Opts.ExtTmpl suffix.
	if len(Opts.TemplateFile) > 0 {
		for _, template := range strings.Split(Opts.TemplateFile, ",") {
			if _, err := os.Stat(template); os.IsNotExist(err) {
				template += Opts.ExtTmpl
			}
			templates = append(templates, template)
		}
	} else if idx := strings.LastIndex(fileName, "."); idx > 0 {
		templates = []string{fileName[:idx] + Opts.ExtTmpl}
	} else {
		templates = []string{fileName + Opts.ExtTmpl}
	}

	m := ReadYamlFile(fileName)

	env := make(map[string]string)
	for _, e := range os.Environ() {
		sep := strings.Index(e, "=")
		env[e[0:sep]] = e[sep+1:]
	}
	m["ENV"] = env
	//fmt.Printf("] %+v\n", m)

	buf := new(bytes.Buffer)
	for _, template := range templates {
		t, err := ParseFiles(HTML, template)
		checkError(err)

		err = t.Execute(buf, m)
		checkError(err)
	}
	return buf.String()
}

// ParseFiles wraps parsing text or HTML template files into a single
// function, dictated by the first parameter "HTML".
// By <NAME> <EMAIL>
func ParseFiles(HTML bool, filenames ...string) (Template, error) {
	var tname string

	if len(Opts.TemplateStr) > 0 {
		tname = "TT"
	} else if len(filenames) == 0 {
		return nil, fmt.Errorf("ParseFiles called without template filename")
	} else {
		tname = filepath.Base(filenames[0])
	}

	textTemplate := tt.New(tname).Funcs(tt.FuncMap{
		"eqf":      strings.EqualFold,
		"split":    strings.Fields,
		"minus1":   minus1,
		"dateI":    dateI,
		"year4":    year4,
		"replace":  replace,
		"replacec": replacec,
	})

	if len(Opts.TemplateStr) > 0 {
		return textTemplate.Parse(Opts.TemplateStr)
	}
	return textTemplate.ParseFiles(filenames...)
}

////////////////////////////////////////////////////////////////////////////
// Version 2 Function definitions

func ReadYamlFile(fileName string) EgData {
	source, err := ioutil.ReadFile(fileName)
	checkError(err)

	m := make(EgData)

	err = yaml.Unmarshal(source, &m)
	checkError(err)

	return m
}

func GetEnv() map[string]string {
	env := make(map[string]string)
	for _, e := range os.Environ() {
		sep := strings.Index(e, "=")
		env[e[0:sep]] = e[sep+1:]
	}
	return env
}

// Process will process the standard easygen input: the `fileName` is for both template and data file names, and produce output from the template according to the corresponding driving data.
func Process(t Template, wr io.Writer, fileName string) error {
	return Process2(t, wr, fileName, fileName)
}

// Process2 will process the case that both template and data file names are given, and produce output according to the given template and driving data files,
// specified via fileNameTempl and fileNames respectively.
// fileNameTempl can be a comma-separated string giving many template files
func Process2(t Template, wr io.Writer, fileNameTempl string, fileNames ...string) error {
}

// Process1 will process the case that both template and data file names are given, and produce output according to the given template and driving data files,
// specified via fileNameTempl and fileName respectively.
// fileNameTempl is not a comma-separated string, but for a single template file.
func Process1(t Template, wr io.Writer, fileNameTempl string, fileName string) error {
	m := ReadYamlFile(fileName + Opts.ExtYaml)
	m["ENV"] = GetEnv()
	//fmt.Printf("] %+v\n", m)

	// template file name
	fileNameT := fileNameTempl
	if len(Opts.TemplateFile) > 0 {
		fileNameT = Opts.TemplateFile
	}
	fileNameT = fileNameT + Opts.ExtTmpl
	tn, err := t.ParseFiles(fileNameT)
	checkError(err)

	return tn.ExecuteTemplate(wr, filepath.Base(fileNameT), m)
}

{'fraction_non_alphanumeric': 0.18252203278148424, 'fraction_numerical': 0.005271394448562721, 'mean_word_length': 2.870576984379981, 'pattern_counts': {'":': 14, '<': 15, '<?xml version=': 0, '>': 25, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5844383', 'n_tokens_mistral': 3670, 'n_tokens_neox': 3593, 'n_words': 1213}