<commit_before>////////////////////////////////////////////////////////////////////////////
// Package: easygen
// Purpose: Easy to use universal code/text generator
// Authors: <NAME> (c) 2015-2021, All rights reserved
////////////////////////////////////////////////////////////////////////////

/*

Package easygen is an easy to use universal code/text generator library.

It can be used as a text or html generator for arbitrary purposes with arbitrary data and templates.

It can be used as a code generator, or anything that is structurally repetitive. Some command line parameter handling code generator are provided as examples, including the Go's built-in flag package, and the viper & cobra package.

Many examples have been provided to showcase its functionality, and different ways to use it.

*/
package easygen

import (
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"path/filepath"
	"regexp"
	"strings"

	"gopkg.in/yaml.v2"
)

////////////////////////////////////////////////////////////////////////////
// Constant and data type/structure definitions

////////////////////////////////////////////////////////////////////////////
// Global variables definitions

// EgData -- EasyGen driven Data
type EgData interface{}

// Opts holds the actual values from the command line parameters
var Opts = Options{ExtYaml: ".yaml", ExtJson: ".json", ExtTmpl: ".tmpl"}

////////////////////////////////////////////////////////////////////////////
// Function definitions

// ReadDataFile reads in the driving data from the given file, which can
// be optionally without the defined extension
func ReadDataFile(fileName string) EgData {
	if IsExist(fileName + Opts.ExtYaml) {
		return ReadYamlFile(fileName + Opts.ExtYaml)
	} else if IsExist(fileName + Opts.ExtJson) {
		return ReadJsonFile(fileName + Opts.ExtJson)
	} else if IsExist(fileName) {
		verbose("Reading exist Data File", 2)
		fext := filepath.Ext(fileName)
		fext = fext[1:] // ignore the leading "."
		if regexp.MustCompile(`(?i)^y`).MatchString(fext) {
			verbose("Reading YAML file", 2)
			return ReadYamlFile(fileName)
		} else if regexp.MustCompile(`(?i)^j`).MatchString(fext) {
			return ReadJsonFile(fileName)
		} else {
			checkError(fmt.Errorf("Unsupported file extension for DataFile '%s'", fileName))
		}
	} else if fileName == "-" {
		// from stdin, prepare temp file
		tempDir, err := ioutil.TempDir("", "easygen-")
		checkError(err)
		defer os.RemoveAll(tempDir)
		file, err := ioutil.TempFile(tempDir, "easygen-*")
		checkError(err)
		defer os.Remove(file.Name())
		//print(file.Name())
		// read from stdin write to temp file
		bytes, err := ioutil.ReadAll(os.Stdin)
		_, err = file.Write(bytes)
		checkError(err)

		// try read it as Json
		if d, err := TryToReadJsonFile(file.Name()); err != nil {
			return d
		}
		// else, read it as Yaml
		return ReadYamlFile(file.Name())
	}
	checkError(fmt.Errorf("DataFile '%s' cannot be found", fileName))
	return nil
}

// ReadYamlFile reads given YAML file as EgData
func ReadYamlFile(fileName string) EgData {
	source, err := ioutil.ReadFile(fileName)
	checkError(err)

	m := make(map[interface{}]interface{})

	err = yaml.Unmarshal(source, &m)
	checkError(err)

	fmt.Printf("] Input %v\n", m)
	return m
}

// ReadJsonFile reads given JSON file as EgData
func ReadJsonFile(fileName string) EgData {
	source, err := ioutil.ReadFile(fileName)
	checkError(err)

	m := make(map[string]interface{})

	err = json.Unmarshal(source, &m)
	checkError(err)

	return m
}

// TryToReadJsonFile will try to read given file as JSON and return error if otherwise
func TryToReadJsonFile(fileName string) (EgData,error) {
	source, err := ioutil.ReadFile(fileName)
	checkError(err)

	m := make(map[string]interface{})

	err = json.Unmarshal(source, &m)
	if err != nil {
		return nil, err
	}

	return m, nil
}

// IsExist checks if the given file exist
func IsExist(fileName string) bool {
	//fmt.Printf("] Checking %s\n", fileName)
	_, err := os.Stat(fileName)
	return err == nil || os.IsExist(err)<commit_msg>- [!] reverse the TryToRead order; OK for both Json and Yaml<commit_after>////////////////////////////////////////////////////////////////////////////
// Package: easygen
// Purpose: Easy to use universal code/text generator
// Authors: <NAME> (c) 2015-2021, All rights reserved
////////////////////////////////////////////////////////////////////////////

/*

Package easygen is an easy to use universal code/text generator library.

It can be used as a text or html generator for arbitrary purposes with arbitrary data and templates.

It can be used as a code generator, or anything that is structurally repetitive. Some command line parameter handling code generator are provided as examples, including the Go's built-in flag package, and the viper & cobra package.

Many examples have been provided to showcase its functionality, and different ways to use it.

*/
package easygen

import (
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"path/filepath"
	"regexp"
	"strings"

	"gopkg.in/yaml.v2"
)

////////////////////////////////////////////////////////////////////////////
// Constant and data type/structure definitions

////////////////////////////////////////////////////////////////////////////
// Global variables definitions

// EgData -- EasyGen driven Data
type EgData interface{}

// Opts holds the actual values from the command line parameters
var Opts = Options{ExtYaml: ".yaml", ExtJson: ".json", ExtTmpl: ".tmpl"}

////////////////////////////////////////////////////////////////////////////
// Function definitions

// ReadDataFile reads in the driving data from the given file, which can
// be optionally without the defined extension
func ReadDataFile(fileName string) EgData {
	if IsExist(fileName + Opts.ExtYaml) {
		return ReadYamlFile(fileName + Opts.ExtYaml)
	} else if IsExist(fileName + Opts.ExtJson) {
		return ReadJsonFile(fileName + Opts.ExtJson)
	} else if IsExist(fileName) {
		verbose("Reading exist Data File", 2)
		fext := filepath.Ext(fileName)
		fext = fext[1:] // ignore the leading "."
		if regexp.MustCompile(`(?i)^y`).MatchString(fext) {
			verbose("Reading YAML file", 2)
			return ReadYamlFile(fileName)
		} else if regexp.MustCompile(`(?i)^j`).MatchString(fext) {
			return ReadJsonFile(fileName)
		} else {
			checkError(fmt.Errorf("Unsupported file extension for DataFile '%s'", fileName))
		}
	} else if fileName == "-" {
		// from stdin, prepare temp file
		tempDir, err := ioutil.TempDir("", "easygen-")
		checkError(err)
		defer os.RemoveAll(tempDir)
		file, err := ioutil.TempFile(tempDir, "easygen-*")
		checkError(err)
		defer os.Remove(file.Name())
		//print(file.Name())
		// read from stdin write to temp file
		bytes, err := ioutil.ReadAll(os.Stdin)
		_, err = file.Write(bytes)
		checkError(err)

		// try read it as Yaml
		if d, err := TryToReadYamlFile(file.Name()); err != nil {
			return d
		}
		// else, read it as Json
		return ReadYamlFile(file.Name())
	}
	checkError(fmt.Errorf("DataFile '%s' cannot be found", fileName))
	return nil
}

// TryToReadYamlFile will try to read given file as YAML and return error if otherwise
func TryToReadYamlFile(fileName string) (EgData,error) {
	source, err := ioutil.ReadFile(fileName)
	checkError(err)

	m := make(map[interface{}]interface{})

	err = yaml.Unmarshal(source, &m)

	if err != nil {
		return nil, err
	}

	return m, nil
}

// ReadYamlFile reads given YAML file as EgData
func ReadYamlFile(fileName string) EgData {
	source, err := ioutil.ReadFile(fileName)
	checkError(err)

	m := make(map[interface{}]interface{})

	err = yaml.Unmarshal(source, &m)
	checkError(err)

	return m
}

// ReadJsonFile reads given JSON file as EgData
func ReadJsonFile(fileName string) EgData {
	source, err := ioutil.ReadFile(fileName)
	checkError(err)

	m := make(map[string]interface{})

	err = json.Unmarshal(source, &m)
	checkError(err)

	fmt.Printf("] Input %v\n", m)
	return m
}

// IsExist checks if the given file exist
func IsExist(fileName string) bool {
	//fmt.Printf("] Checking %s\n", fileName)
	_, err := os.Stat(fileName)
	return err == nil || os.IsExist(err)
{'fraction_non_alphanumeric': 0.20237066304482035, 'fraction_numerical': 0.002963328806025435, 'mean_word_length': 2.6885245901639343, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1482860', 'n_tokens_mistral': 2554, 'n_tokens_neox': 2436, 'n_words': 783}